Merge {DS-DRR improvements:code cleanup} into MWL#128+DS-MRR tree

sql/multi_range_read.cc

@@ -282,8 +282,20 @@ scan_it_again:
   DBUG_RETURN(result);
 }
 
+/****************************************************************************
+ * Mrr_*_reader classes (building blocks for DS-MRR)
+ ***************************************************************************/
 
-/***** MRR_impl classes ****************************************************/
+int Mrr_simple_index_reader::init(handler *h_arg, RANGE_SEQ_IF *seq_funcs, 
+                                  void *seq_init_param, uint n_ranges,
+                                  uint mode, Buffer_manager *buf_manager_arg)
+{
+  HANDLER_BUFFER no_buffer = {NULL, NULL, NULL};
+  h= h_arg;
+  res= 0;
+  return h->handler::multi_range_read_init(seq_funcs, seq_init_param, n_ranges,
+                                           mode, &no_buffer);
+}
 
 int Mrr_simple_index_reader::get_next(char **range_info)
 {
@@ -297,44 +309,35 @@ int Mrr_simple_index_reader::get_next(char **range_info)
   return res;
 }
 
-int Mrr_simple_index_reader::init(handler *h_arg, RANGE_SEQ_IF *seq_funcs, 
-                                  void *seq_init_param, uint n_ranges,
-                                  uint mode, Buffer_manager *buf_manager_arg)
-{
-  HANDLER_BUFFER no_buffer = {NULL, NULL, NULL};
-  h= h_arg;
-  res= 0;
-  return h->handler::multi_range_read_init(seq_funcs, seq_init_param, n_ranges,
-                                           mode, &no_buffer);
-}
 
 /**
-  DS-MRR/CPK: multi_range_read_next() function
+  @brief Get next index record
 
   @param range_info  OUT identifier of range that the returned record belongs to
   
   @note
-    This function walks over key buffer and does index reads, i.e. it produces
-    {current_record, range_id} pairs.
-
-    The function has the same call contract like multi_range_read_next()'s.
-
     We actually iterate over nested sequences:
-    - a disjoint sequence of index ranges
+    - an ordered sequence of groups of identical keys
-      - each range has multiple records
+      - each key group has key value, which has multiple matching records 
-        - each record goes into multiple identical ranges.
+        - thus, each record matches all members of the key group
 
   @retval 0                   OK, next record was successfully read
   @retval HA_ERR_END_OF_FILE  End of records
   @retval Other               Some other error
 */
 
-int Mrr_ordered_index_reader::get_next(char **range_info_arg)
+int Mrr_ordered_index_reader::get_next(char **range_info)
 {
   DBUG_ENTER("Mrr_ordered_index_reader::get_next");
   
-  if (!know_key_tuple_params) /* We're in startup phase */
+  if (!know_key_tuple_params)
+  {
+    /* 
+      We're at the very start, haven't filled the buffer or even know what
+      will be there. Force the caller to call refill_buffer():
+    */
     DBUG_RETURN(HA_ERR_END_OF_FILE);
+  }
 
   while (1)
   {
@@ -354,27 +357,11 @@ int Mrr_ordered_index_reader::get_next(char **range_info_arg)
       while (kv_it.init(this))
       {
         if (key_buffer->is_empty())
-        {
-          /*if (auto_refill)
-          {
-            int res;
-            if ((res= refill_buffer()))
-              DBUG_RETURN(res);
-            if (key_buffer->is_empty())
         {
           index_scan_eof= TRUE;
           DBUG_RETURN(HA_ERR_END_OF_FILE);
         }
       }
-          else
-          */
-          {
-            /* Buffer refills are managed by somebody else for us */
-            index_scan_eof= TRUE;
-            DBUG_RETURN(HA_ERR_END_OF_FILE);
-          }
-        }
-      }
       scanning_key_val_iter= TRUE;
     }
 
@@ -390,24 +377,21 @@ int Mrr_ordered_index_reader::get_next(char **range_info_arg)
     /* Go get another (record, range_id) combination */
   } /* while */
 
-  memcpy(range_info_arg, cur_range_info, sizeof(void*));
+  memcpy(range_info, cur_range_info, sizeof(void*));
   DBUG_RETURN(0);
 }
 
 
 /**
-  DS-MRR/CPK: Fill the buffer with (lookup_tuple, range_id) pairs and sort
+  Fill the buffer with (lookup_tuple, range_id) pairs and sort
-  
-  Enumerate the input range (=key) sequence, fill the key buffer with 
-  (lookup_key, range_id) pairs and sort it.
-
-  When this function returns, either
-   - key buffer is non-empty, or
-   - key buffer is empty and source range sequence is exhausted
  
   @note
-    dsmrr_eof is set to indicate whether we've exhausted the list of ranges 
+    We don't know lookup_tuple before we get the first key from
-    we're scanning.
+    mrr_funcs.get_next(). Not knowing tuple length means we can't setup the
+    key buffer (in particular, which part of the buffer space it should occupy
+    when we have both key and rowid buffers).  This problem is solved by having 
+    know_key_tuple_params variabe, and buf_manager, which we ask to set/reset
+    buffers for us.
 */
 
 int Mrr_ordered_index_reader::refill_buffer()
@@ -427,23 +411,7 @@ int Mrr_ordered_index_reader::refill_buffer()
                               is_mrr_assoc? (uchar**)&range_info_ptr : NULL,
                               sizeof(uchar*));
   }
-#if 0
 
-  if (know_key_tuple_params)
-  {
-    if (do_rndpos_scan && rowid_buffer.is_empty())
-    {
-      /*
-        We're using two buffers and both of them are empty now. Restore the
-        original sizes
-      */
-      rowid_buffer.set_buffer_space(full_buf, rowid_buffer_end);
-      key_buffer= &backward_key_buf;
-      key_buffer->set_buffer_space(rowid_buffer_end, full_buf_end);
-    }
-  }
-  is all of the ifdef-ed stuff is handled above?
-#endif
   while ((!know_key_tuple_params || key_buffer->can_write()) && 
          !(res= mrr_funcs.next(mrr_iter, &cur_range)))
   {
@@ -478,7 +446,7 @@ int Mrr_ordered_index_reader::refill_buffer()
     key_buffer->write();
   }
 
-  no_more_keys= test(res);
+  bool no_more_keys= test(res);
   scanning_key_val_iter= FALSE;
   index_scan_eof= FALSE;
 
@@ -523,14 +491,7 @@ int Mrr_ordered_rndpos_reader::init(handler *h_arg,
   index_reader= index_reader_arg;
   rowid_buffer= buf;
   is_mrr_assoc= !test(mode & HA_MRR_NO_ASSOCIATION);
-  //rowid_buff_elem_size= h->ref_length;
-  //if (!(mode & HA_MRR_NO_ASSOCIATION))
-  //  rowid_buff_elem_size += sizeof(char*);
-
   index_reader_exhausted= FALSE;
-  ///int res= index_reader->refill_buffer();
-  ///if (res && res!=HA_ERR_END_OF_FILE)
-  ///  return res;
   return 0;
 }
 
@@ -553,7 +514,6 @@ int Mrr_ordered_rndpos_reader::init(handler *h_arg,
   @retval other  Error
 */
 
-
 int Mrr_ordered_rndpos_reader::refill_buffer()
 {
   int res;
@@ -562,7 +522,7 @@ int Mrr_ordered_rndpos_reader::refill_buffer()
   if (index_reader_exhausted)
     DBUG_RETURN(HA_ERR_END_OF_FILE);
 
-  while ((res= refill2() == HA_ERR_END_OF_FILE))
+  while ((res= refill_from_key_buffer() == HA_ERR_END_OF_FILE))
   {
     if ((res= index_reader->refill_buffer()))
     {
@@ -575,13 +535,23 @@ int Mrr_ordered_rndpos_reader::refill_buffer()
 }
 
 
-/* This one refills without calling index_reader->refill_buffer(). */
+void Mrr_index_reader::position()
-int Mrr_ordered_rndpos_reader::refill2()
+{
+  h->position(h->get_table()->record[0]);
+}
+
+
+/* 
+  @brief Try to refill the rowid buffer without calling
+  index_reader->refill_buffer(). 
+*/
+
+int Mrr_ordered_rndpos_reader::refill_from_key_buffer()
 {
   char *range_info;
   uchar **range_info_ptr= (uchar**)&range_info;
   int res;
-  DBUG_ENTER("Mrr_ordered_rndpos_reader::refill2");
+  DBUG_ENTER("Mrr_ordered_rndpos_reader::refill_from_key_buffer");
 
   DBUG_ASSERT(rowid_buffer->is_empty());
   index_rowid= index_reader->get_rowid_ptr();
@@ -600,7 +570,7 @@ int Mrr_ordered_rndpos_reader::refill2()
       break;
 
     /* Put rowid, or {rowid, range_id} pair into the buffer */
-    index_reader->h->position(index_reader->h->get_table()->record[0]);
+    index_reader->position();
 
     rowid_buffer->write();
   }
@@ -615,10 +585,12 @@ int Mrr_ordered_rndpos_reader::refill2()
 }
 
 
-/**
+/*
-  DS-MRR implementation: multi_range_read_next() function.
+  Get the next record+range_id using ordered array of rowid+range_id pairds
 
-  Calling convention is like multi_range_read_next() has.
+  @note
+    Since we have sorted rowids, we try not to make multiple rnd_pos() calls
+    with the same rowid value.
 */
 
 int Mrr_ordered_rndpos_reader::get_next(char **range_info)
@@ -652,31 +624,6 @@ int Mrr_ordered_rndpos_reader::get_next(char **range_info)
 
   while (1)
   {
-#if 0      
-    if (rowid_buffer->is_empty()) /* We're out of rowids */
-    {
-      /* First, finish off the sorted keys we have */ 
-      if (!index_reader->eof())
-      {
-        res= refill_buffer();
-        if (res && res != HA_ERR_END_OF_FILE)
-          return res;
-      }
-
-      if (rowid_buffer->is_empty())
-      {
-        /*
-          Ok neither index_reader nor us have any records. Refill index
-          reader, then refill us.
-        */
-        // TODO: if key buffer is empty, too, redistribute the buffer space.
-        if ((res= index_reader->refill_buffer()) ||
-            (res= refill_buffer()))
-          return res;
-      }
-    }
-#endif
-   
     last_identical_rowid= NULL;
 
     /* Return eof if there are no rowids in the buffer after re-fill attempt */
@@ -724,13 +671,8 @@ int Mrr_ordered_rndpos_reader::get_next(char **range_info)
 }
 
 
-
-
-/************ MRR_impl classes end *********************************************/
-
-
 /****************************************************************************
- * DS-MRR implementation 
+ * Top-level DS-MRR implementation functions (the ones called by storage engine)
  ***************************************************************************/
 
 /**
@@ -769,7 +711,7 @@ int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs,
   if ((mode & HA_MRR_USE_DEFAULT_IMPL) || (mode & HA_MRR_SORTED))
   {
     DBUG_ASSERT(h->inited == handler::INDEX);
-    Mrr_simple_index_reader *s= &strategy_factory.simple_index_reader;
+    Mrr_simple_index_reader *s= &reader_factory.simple_index_reader;
     res= s->init(h, seq_funcs, seq_init_param, n_ranges, mode, this);
     strategy= s;
     DBUG_RETURN(res);
@@ -786,10 +728,10 @@ int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs,
   if ((mode & HA_MRR_SINGLE_POINT) &&
       optimizer_flag(thd, OPTIMIZER_SWITCH_MRR_SORT_KEYS))
   {
-    index_strategy= ordered_idx_reader= &strategy_factory.ordered_index_reader;
+    index_strategy= ordered_idx_reader= &reader_factory.ordered_index_reader;
   }
   else
-    index_strategy= &strategy_factory.simple_index_reader;
+    index_strategy= &reader_factory.simple_index_reader;
 
   strategy= index_strategy;
   /*
@@ -807,7 +749,7 @@ int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs,
   Mrr_ordered_rndpos_reader *disk_strategy= NULL;
   if (!(keyno == table->s->primary_key && h_idx->primary_key_is_clustered()))
   {
-    strategy= disk_strategy= &strategy_factory.ordered_rndpos_reader;
+    strategy= disk_strategy= &reader_factory.ordered_rndpos_reader;
   }
 
   if (is_mrr_assoc)
@@ -818,8 +760,6 @@ int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs,
   
   if (strategy == index_strategy)
   {
-    ///if (ordered_idx_reader)
-    //  ordered_idx_reader->auto_refill= TRUE;
     /* Index strategy serves it all. We don't need two handlers, etc */
     /* Give the buffer to index strategy */
     if ((res= index_strategy->init(h, seq_funcs, seq_init_param, n_ranges,
@@ -838,9 +778,6 @@ int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs,
     if ((res= setup_two_handlers()))
       DBUG_RETURN(res);
 
-    ///if (ordered_idx_reader)
-    ///  ordered_idx_reader->auto_refill= FALSE;
-
     if ((res= index_strategy->init(h2, seq_funcs, seq_init_param, n_ranges, 
                                    mode, this)) || 
         (res= disk_strategy->init(h, index_strategy, mode, &rowid_buffer)))
@@ -960,8 +897,8 @@ int DsMrr_impl::setup_two_handlers()
       goto error;
   }
   DBUG_RETURN(0);
+
 error:
-  //close_second_handler(); -- caller does that
   DBUG_RETURN(res);
 }
 
@@ -1132,6 +1069,7 @@ void DsMrr_impl::reset_buffer_sizes()
   }
 }
 
+
 /**
   Take unused space from the key buffer and give it to the rowid buffer
 */
@@ -1144,14 +1082,9 @@ void DsMrr_impl::reallocate_buffer_space()
 }
 
 
-//////////////////////////////////////////////////////////////////////////////
-//////////////////////////////////////////////////////////////////////////////
-
 bool Key_value_records_iterator::init(Mrr_ordered_index_reader *owner_arg)
 {
   int res;
-  //h= h_arg;
-  //param= param_arg;
   owner= owner_arg;
 
   identical_key_it.init(owner->key_buffer);

sql/multi_range_read.h

@@ -49,7 +49,10 @@
 #include "sql_lifo_buffer.h"
 
 class DsMrr_impl;
+class Mrr_ordered_index_reader;
 
+
+/* A structure with key parameters that's shared among several classes */
 class Key_parameters
 {
 public:
@@ -70,72 +73,68 @@ public:
   bool use_key_pointers;
 };
 
-/**
-  Iterator over (record, range_id) pairs that match given key value.
 
-  We may need to scan multiple (key_val, range_id) pairs with the same 
+/**
-  key value. A key value may have multiple matching records, so we'll need to
+  A class to enumerate (record, range_id) pairs that match given key value.
-  produce a cross-product of sets of matching records and range_id-s.
+  
+  The idea is that we have an array of
+
+    (key, range_id1), (key, range_id2) ... (key, range_idN)
+
+  pairs, i.e. multiple identical key values with their different range_id-s,
+  and also we have ha_engine object where we can find matches for the key
+  value.
+
+  What this class does is produces all combinations of (key_match_record_X,
+  range_idN) pairs.
 */
-class Mrr_ordered_index_reader;
+
 class Key_value_records_iterator
 {
-  /* Scan parameters */
+  /* Use this to get table handler, key buffer and other parameters */
-  Key_parameters *param;
+  Mrr_ordered_index_reader *owner;
   Lifo_buffer_iterator identical_key_it;
+
   uchar *last_identical_key_ptr;
   bool get_next_row;
-  //handler *h;
-  /* TRUE <=> We can get at most one index tuple for a lookup key */
-  //bool index_ranges_unique;
   
-  Mrr_ordered_index_reader *owner;
+  uchar *cur_index_tuple; /* key_buffer.read() reads to here */
-  /* key_buffer.read() reads to here */
-  uchar *cur_index_tuple;
 public:
   bool init(Mrr_ordered_index_reader *owner_arg);
-
-  /*
-    Get next (key_val, range_id) pair.
-  */
   int get_next();
-
   void close();
-  friend class Mrr_ordered_index_reader;
 };
 
 
 /*
-  Something that will manage buffers for those that call it
+  Buffer manager interface. Mrr_reader objects use it to inqure DsMrr_impl
+  to manage buffer space for them.
 */
 class Buffer_manager
 {
 public:
-  virtual void reset_buffer_sizes()= 0;
   virtual void setup_buffer_sizes(uint key_size_in_keybuf, 
                                   key_part_map key_tuple_map)=0;
+  virtual void reset_buffer_sizes()= 0;
   virtual Lifo_buffer* get_key_buffer()= 0;
-  virtual ~Buffer_manager(){}
+  virtual ~Buffer_manager(){} /* Shut up the compiler */
 };
 
 
 /* 
-  Abstract MRR execution strategy
+  Mrr_reader - DS-MRR execution strategy abstraction
 
-  An object of this class produces (R, range_info) pairs where R can be an
+  A reader produces ([index]_record, range_info) pairs, and requires periodic
-  index tuple or a table record.
+  refill operations.
 
-  Getting HA_ERR_END_OF_FILE from get_next() means that the source should be
+  - one starts using the reader by calling reader->get_next(),
-  re-filled. 
+  - when a get_next() call returns HA_ERR_END_OF_FILE, one must call 
-  
+    refill_buffer() before they can make more get_next() calls.
-  Was:
+  - when refill_buffer() returns HA_ERR_END_OF_FILE, this means the real
-  if eof() returns true after refill attempt, then the end of 
+    end of stream and get_next() should not be called anymore.
-  stream has been reached and get_next() must not be called anymore.
-
-  Now:
-  if refill_buffer() returns HA_ERR_END_OF_FILE that means the stream is 
-  really exhausted.
 
+  Both functions can return other error codes, these mean unrecoverable errors
+  after which one cannot continue.
 */
 
 class Mrr_reader 
@@ -143,28 +142,33 @@ class Mrr_reader
 public:
   virtual int get_next(char **range_info) = 0;
   virtual int refill_buffer() = 0;
-  
   virtual ~Mrr_reader() {}; /* just to remove compiler warning */
 };
 
 
-/* A common base for strategies that do index scans and produce index tuples */
+/* 
+  A common base for readers that do index scans and produce index tuples 
+*/
+
 class Mrr_index_reader : public Mrr_reader
 {
+protected:
+  handler *h; /* Handler object to use */
 public:
-  handler *h;
-
   virtual int init(handler *h_arg, RANGE_SEQ_IF *seq_funcs, 
                    void *seq_init_param, uint n_ranges,
                    uint mode, Buffer_manager *buf_manager_arg) = 0;
-  virtual bool eof() = 0; 
+
+  /* Get pointer to place where every get_next() call will put rowid */
   virtual uchar *get_rowid_ptr()= 0;
+  /* Get the rowid (call this after get_next() call) */
+  void position();
   virtual bool skip_record(char *range_id, uchar *rowid)=0;
 };
 
 
 /*
-  A "bypass" strategy that uses default MRR implementation (i.e.
+  A "bypass" reader that uses default MRR implementation (i.e.
   handler::multi_range_read_XXX() calls) to produce rows.
 */
 
@@ -177,7 +181,6 @@ public:
            uint mode, Buffer_manager *buf_manager_arg);
   int get_next(char **range_info);
   int refill_buffer() { return HA_ERR_END_OF_FILE; }
-  bool eof() { return test(res); }
   uchar *get_rowid_ptr() { return h->ref; }
   bool skip_record(char *range_id, uchar *rowid)
   {
@@ -187,9 +190,8 @@ public:
 };
 
 
-
 /* 
-  A strategy that sorts index lookup keys before scanning the index
+  A reader that sorts the key values before it makes the index lookups.
 */
 
 class Mrr_ordered_index_reader : public Mrr_index_reader
@@ -200,7 +202,6 @@ public:
            uint mode, Buffer_manager *buf_manager_arg);
   int get_next(char **range_info);
   int refill_buffer();
-  bool eof() { return index_scan_eof; }
   uchar *get_rowid_ptr() { return h->ref; }
   
   bool skip_record(char *range_info, uchar *rowid)
@@ -223,23 +224,18 @@ private:
   /* Initially FALSE, becomes TRUE when we've set key_tuple_xxx members */
   bool know_key_tuple_params;
 
- // bool use_key_pointers;
-  
   Key_parameters  keypar;
+
   /* TRUE <=> need range association, buffers hold {rowid, range_id} pairs */
   bool is_mrr_assoc;
 
-  bool no_more_keys;
   RANGE_SEQ_IF mrr_funcs;
   range_seq_t mrr_iter;
 
-  //bool auto_refill;
-
   bool index_scan_eof;
 
   static int key_tuple_cmp(void* arg, uchar* key1, uchar* key2);
   static int key_tuple_cmp_reverse(void* arg, uchar* key1, uchar* key2);
-  //void cleanup();
   
   friend class Key_value_records_iterator; 
   friend class DsMrr_impl;
@@ -247,7 +243,10 @@ private:
 };
 
 
-/* MRR strategy that fetches rowids */
+/* 
+  A reader that gets rowids from an Mrr_index_reader, and then sorts them 
+  before getting full records with handler->rndpos() calls.
+*/
 
 class Mrr_ordered_rndpos_reader : public Mrr_reader 
 {
@@ -256,8 +255,6 @@ public:
            Lifo_buffer *buf);
   int get_next(char **range_info);
   int refill_buffer();
-  int refill2();
-  void cleanup();
 private:
   handler *h;
   
@@ -273,14 +270,15 @@ private:
   uchar *last_identical_rowid;
   Lifo_buffer *rowid_buffer;
   
-  /* = h->ref_length  [ + sizeof(range_assoc_info) ] */
-  //uint rowid_buff_elem_size;
-
   /* rowid_buffer.read() will set the following:  */
   uchar *rowid;
   uchar *rowids_range_id;
+
+  int refill_from_key_buffer();
 };
 
+
+/* A place where one can get readers without having to alloc them on the heap */
 class Mrr_reader_factory
 {
 public:
@@ -289,6 +287,7 @@ public:
   Mrr_simple_index_reader   simple_index_reader;
 };
 
+
 /*
   DS-MRR implementation for one table. Create/use one object of this class for
   each ha_{myisam/innobase/etc} object. That object will be further referred to
@@ -458,17 +457,11 @@ private:
   */
   handler *h2;
   
-  /** Properties of current MRR scan **/
-
   uint keyno; /* index we're running the scan on */
   /* TRUE <=> need range association, buffers hold {rowid, range_id} pairs */
   bool is_mrr_assoc;
-  /* TRUE <=> sort the keys before making index lookups */
-  //bool do_sort_keys;
-  /* TRUE <=> sort rowids and use rnd_pos() to get and return full records */
-  //bool do_rndpos_scan;
 
-  Mrr_reader_factory strategy_factory;
+  Mrr_reader_factory reader_factory;
   Mrr_reader *strategy;
   Mrr_index_reader *index_strategy;
 
@@ -484,8 +477,6 @@ private:
   */
   uchar *rowid_buffer_end;
  
-  /** Index scaning and key buffer-related members **/
-
   /*
     One of the following two is used for key buffer: forward is used when 
     we only need key buffer, backward is used when we need both key and rowid
@@ -494,18 +485,11 @@ private:
   Forward_lifo_buffer forward_key_buf;
   Backward_lifo_buffer backward_key_buf;
 
-  Forward_lifo_buffer rowid_buffer;
-  
-  /* = key_size_in_keybuf [ + sizeof(range_assoc_info) ] */
-  //uint key_buff_elem_size_;
-  
-  /** rnd_pos() scan and rowid buffer-related members **/
-
   /*
     Buffer to store (rowid, range_id) pairs, or just rowids if 
     is_mrr_assoc==FALSE
   */
-  //Forward_lifo_buffer rowid_buffer;
+  Forward_lifo_buffer rowid_buffer;
   
   bool choose_mrr_impl(uint keyno, ha_rows rows, uint *flags, uint *bufsz, 
                        COST_VECT *cost);