<commit_before>#include "mysql_priv.h"
#include "sql_select.h"

/****************************************************************************
 * Default MRR implementation (MRR to non-MRR converter)
 ***************************************************************************/

/**
  Get cost and other information about MRR scan over a known list of ranges

  Calculate estimated cost and other information about an MRR scan for given
  sequence of ranges.

  @param keyno           Index number
  @param seq             Range sequence to be traversed
  @param seq_init_param  First parameter for seq->init()
  @param n_ranges_arg    Number of ranges in the sequence, or 0 if the caller
                         can't efficiently determine it
  @param bufsz    INOUT  IN:  Size of the buffer available for use
                         OUT: Size of the buffer that is expected to be actually
                              used, or 0 if buffer is not needed.
  @param flags    INOUT  A combination of HA_MRR_* flags
  @param cost     OUT    Estimated cost of MRR access

  @note
    This method (or an overriding one in a derived class) must check for
    thd->killed and return HA_POS_ERROR if it is not zero. This is required
    for a user to be able to interrupt the calculation by killing the
    connection/query.

  @retval
    HA_POS_ERROR  Error or the engine is unable to perform the requested
                  scan. Values of OUT parameters are undefined.
  @retval
    other         OK, *cost contains cost of the scan, *bufsz and *flags
                  contain scan parameters.
*/

ha_rows 
handler::multi_range_read_info_const(uint keyno, RANGE_SEQ_IF *seq,
                                     void *seq_init_param, uint n_ranges_arg,
                                     uint *bufsz, uint *flags, COST_VECT *cost)
{
  KEY_MULTI_RANGE range;
  range_seq_t seq_it;
  ha_rows rows, total_rows= 0;
  uint n_ranges=0;
  THD *thd= current_thd;
  
  /* Default MRR implementation doesn't need buffer */
  *bufsz= 0;

  seq_it= seq->init(seq_init_param, n_ranges, *flags);
  while (!seq->next(seq_it, &range))
  {
    if (unlikely(thd->killed != 0))
      return HA_POS_ERROR;
    
    n_ranges++;
    key_range *min_endp, *max_endp;
    if (range.range_flag & GEOM_FLAG)
    {
      /* In this case tmp_min_flag contains the handler-read-function */
      range.start_key.flag= (ha_rkey_function) (range.range_flag ^ GEOM_FLAG);
      min_endp= &range.start_key;
      max_endp= NULL;
    }
    else
    {
      min_endp= range.start_key.length? &range.start_key : NULL;
      max_endp= range.end_key.length? &range.end_key : NULL;
    }
    if ((range.range_flag & UNIQUE_RANGE) && !(range.range_flag & NULL_RANGE))
      rows= 1; /* there can be at most one row */
    else
    {
      if (HA_POS_ERROR == (rows= this->records_in_range(keyno, min_endp, 
                                                        max_endp)))
      {
        /* Can't scan one range => can't do MRR scan at all */
        total_rows= HA_POS_ERROR;
        break;
      }
    }
    total_rows += rows;
  }
  
  if (total_rows != HA_POS_ERROR)
  {
    /* The following calculation is the same as in multi_range_read_info(): */
    *flags |= HA_MRR_USE_DEFAULT_IMPL;
    cost->zero();
    cost->avg_io_cost= 1; /* assume random seeks */
    if ((*flags & HA_MRR_INDEX_ONLY) && total_rows > 2)
      cost->io_count= index_only_read_time(keyno, (uint)total_rows);
    else
      cost->io_count= read_time(keyno, n_ranges, total_rows);
    cost->cpu_cost= (double) total_rows / TIME_FOR_COMPARE + 0.01;
  }
  return total_rows;
}


/**
  Get cost and other information about MRR scan over some sequence of ranges

  Calculate estimated cost and other information about an MRR scan for some
  sequence of ranges.

  The ranges themselves will be known only at execution phase. When this
  function is called we only know number of ranges and a (rough) E(#records)
  within those ranges.

  Currently this function is only called for "n-keypart singlepoint" ranges,
  i.e. each range is "keypart1=someconst1 AND ... AND keypartN=someconstN"

  The flags parameter is a combination of those flags: HA_MRR_SORTED,
  HA_MRR_INDEX_ONLY, HA_MRR_NO_ASSOCIATION, HA_MRR_LIMITS.

  @param keyno           Index number
  @param n_ranges        Estimated number of ranges (i.e. intervals) in the
                         range sequence.
  @param n_rows          Estimated total number of records contained within all
                         of the ranges
  @param bufsz    INOUT  IN:  Size of the buffer available for use
                         OUT: Size of the buffer that will be actually used, or
                              0 if buffer is not needed.
  @param flags    INOUT  A combination of HA_MRR_* flags
  @param cost     OUT    Estimated cost of MRR access

  @retval
    0     OK, *cost contains cost of the scan, *bufsz and *flags contain scan
          parameters.
  @retval
    other Error or can't perform the requested scan
*/

ha_rows handler::multi_range_read_info(uint keyno, uint n_ranges, uint n_rows,
                                       uint key_parts, uint *bufsz, 
                                       uint *flags, COST_VECT *cost)
{
  *bufsz= 0; /* Default implementation doesn't need a buffer */
  //psergey2-todo: assert for singlepoint ranges here?
  *flags |= HA_MRR_USE_DEFAULT_IMPL;

  cost->zero();
  cost->avg_io_cost= 1; /* assume random seeks */

  /* Produce the same cost as non-MRR code does */
  if (*flags & HA_MRR_INDEX_ONLY)
    cost->io_count= index_only_read_time(keyno, n_rows);
  else
    cost->io_count= read_time(keyno, n_ranges, n_rows);
  return 0;
}


/**
  Initialize the MRR scan

  Initialize the MRR scan. This function may do heavyweight scan 
  initialization like row prefetching/sorting/etc (NOTE: but better not do
  it here as we may not need it, e.g. if we never satisfy WHERE clause on
  previous tables. For many implementations it would be natural to do such
  initializations in the first multi_read_range_next() call)

  mode is a combination of the following flags: HA_MRR_SORTED,
  HA_MRR_INDEX_ONLY, HA_MRR_NO_ASSOCIATION 

  @param seq             Range sequence to be traversed
  @param seq_init_param  First parameter for seq->init()
  @param n_ranges        Number of ranges in the sequence
  @param mode            Flags, see the description section for the details
  @param buf             INOUT: memory buffer to be used

  @note
    One must have called index_init() before calling this function. Several
    multi_range_read_init() calls may be made in course of one query.

    Until WL#2623 is done (see its text, section 3.2), the following will 
    also hold:
    The caller will guarantee that if "seq->init == mrr_ranges_array_init"
    then seq_init_param is an array of n_ranges KEY_MULTI_RANGE structures.
    This property will only be used by NDB handler until WL#2623 is done.
     
    Buffer memory management is done according to the following scenario:
    The caller allocates the buffer and provides it to the callee by filling
    the members of HANDLER_BUFFER structure.
    The callee consumes all or some fraction of the provided buffer space, and
    sets the HANDLER_BUFFER members accordingly.
    The callee may use the buffer memory until the next multi_range_read_init()
    call is made, all records have been read, or until index_end() call is
    made, whichever comes first.

  @retval 0  OK
  @retval 1  Error
*/

int
handler::multi_range_read_init(RANGE_SEQ_IF *seq_funcs, void *seq_init_param,
                               uint n_ranges, uint key_parts, uint mode, 
                               HANDLER_BUFFER *buf)
{
  DBUG_ENTER("handler::multi_range_read_init");
  mrr_iter= seq_funcs->init(seq_init_param, n_ranges, mode);
  mrr_funcs= *seq_funcs;
  mrr_is_output_sorted= test(mode & HA_MRR_SORTED);
  mrr_have_range= FALSE;
  DBUG_RETURN(0);
}


/**
  Get next record in MRR scan

  Default MRR implementation: read the next record

  @param range_info  OUT  Undefined if HA_MRR_NO_ASSOCIATION flag is in effect
                          Otherwise, the opaque value associated with the range
                          that contains the returned record.

  @retval 0      OK
  @retval other  Error code
*/

int handler::multi_range_read_next(char **range_info)
{
  int UNINIT_VAR(result);
  int range_res;
  DBUG_ENTER("handler::multi_range_read_next");

  if (!mrr_have_range)
  {
    mrr_have_range= TRUE;
    goto start;
  }

  do
  {
    /* Save a call if there can be only one row in range. */
    if (mrr_cur_range.range_flag != (UNIQUE_RANGE | EQ_RANGE))
    {
      result= read_range_next();
      /* On success or non-EOF errors jump to the end. */
      if (result != HA_ERR_END_OF_FILE)
        break;
    }
    else
    {
      if (was_semi_consistent_read())
        goto scan_it_again;
      /*
        We need to set this for the last range only, but checking this
        condition is more expensive than just setting the result code.
      */
      result= HA_ERR_END_OF_FILE;
    }

start:
    /* Try the next range(s) until one matches a record. */
    while (!(range_res= mrr_funcs.next(mrr_iter, &mrr_cur_range)))
    {
scan_it_again:
      result= read_range_first(mrr_cur_range.start_key.keypart_map ?
                                 &mrr_cur_range.start_key : 0,
                               mrr_cur_range.end_key.keypart_map ?
                                 &mrr_cur_range.end_key : 0,
                               test(mrr_cur_range.range_flag & EQ_RANGE),
                               mrr_is_output_sorted);
      if (result != HA_ERR_END_OF_FILE)
        break;
    }
  }
  while ((result == HA_ERR_END_OF_FILE) && !range_res);

  *range_info= mrr_cur_range.ptr;
  DBUG_PRINT("exit",("handler::multi_range_read_next result %d", result));
  DBUG_RETURN(result);
}

/****************************************************************************
 * DS-MRR implementation 
 ***************************************************************************/

/**
  DS-MRR: Initialize and start MRR scan

  Initialize and start the MRR scan. Depending on the mode parameter, this
  may use default or DS-MRR implementation.

  @param h               Table handler to be used
  @param key             Index to be used
  @param seq_funcs       Interval sequence enumeration functions
  @param seq_init_param  Interval sequence enumeration parameter
  @param n_ranges        Number of ranges in the sequence.
  @param mode            HA_MRR_* modes to use
  @param buf             INOUT Buffer to use

  @retval 0     Ok, Scan started.
  @retval other Error
*/

int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs, 
                           void *seq_init_param, uint n_ranges, uint key_parts,
                           uint mode,
                           HANDLER_BUFFER *buf)
{
  uint elem_size;
  Item *pushed_cond= NULL;
  handler *new_h2= 0;
  DBUG_ENTER("DsMrr_impl::dsmrr_init");

  /*
    index_merge may invoke a scan on an object for which dsmrr_info[_const]
    has not been called, so set the owner handler here as well.
  */
  h= h_arg;
  if (mode & HA_MRR_USE_DEFAULT_IMPL || mode & HA_MRR_SORTED)
  {
    use_default_impl= TRUE;
    const int retval=
      h->handler::multi_range_read_init(seq_funcs, seq_init_param,
                                        n_ranges, key_parts, mode, buf);
    DBUG_RETURN(retval);
  }
  rowids_buf= buf->buffer;

  is_mrr_assoc= !test(mode & HA_MRR_NO_ASSOCIATION);

  if (is_mrr_assoc)
    status_var_increment(table->in_use->status_var.ha_multi_range_read_init_count);
 
  rowids_buf_end= buf->buffer_end;


  doing_cpk_scan= check_cpk_scan(h->active_index, mode); 
  if (doing_cpk_scan)
  {
    uint keylen=0;
    DBUG_ASSERT(key_parts != 0);
    //psergey2-todo: new elem_size here
    for (uint kp= 0; kp < key_parts; kp++)
      keylen += table->key_info[h->active_index].key_part[kp].store_length;

    cpk_tuple_length= keylen;
    cpk_is_unique_scan= test(table->key_info[h->active_index].key_parts == 
                             key_parts);
    cpk_have_range= FALSE;
    elem_size= keylen + (int)is_mrr_assoc * sizeof(void*);
    use_default_impl= FALSE;
  }
  else
    elem_size= h->ref_length + (int)is_mrr_assoc * sizeof(void*);

  rowids_buf_last= rowids_buf + 
                      ((rowids_buf_end - rowids_buf)/ elem_size)*
                      elem_size;
  rowids_buf_end= rowids_buf_last;

  /*
    There can be two cases:
    - This is the first call since index_init(), h2==NULL
       Need to setup h2 then.
    - This is not the first call, h2 is initalized and set up appropriately.
       The caller might have called h->index_init(), need to switch h to
       rnd_pos calls.
  */
  //psergey2-todo: don't create secondary for CPK scan.
  if (!doing_cpk_scan)
  {
    if (!h2)
    {
      /* Create a separate handler object to do rndpos() calls. */
      THD *thd= current_thd;
      /*
        ::clone() takes up a lot of stack, especially on 64 bit platforms.
        The constant 5 is an empiric result.
      */
      if (check_stack_overrun(thd, 5*STACK_MIN_SIZE, (uchar*) &new_h2))
        DBUG_RETURN(1);
      DBUG_ASSERT(h->active_index != MAX_KEY);
      uint mrr_keyno= h->active_index;

      /* Create a separate handler object to do rndpos() calls. */
      if (!(new_h2= h->clone(thd->mem_root)) || 
          new_h2->ha_external_lock(thd, F_RDLCK))
      {
        delete new_h2;
        DBUG_RETURN(1);
      }

      if (mrr_keyno == h->pushed_idx_cond_keyno)
        pushed_cond= h->pushed_idx_cond;

      /*
        Caution: this call will invoke this->dsmrr_close(). Do not put the
        created secondary table handler into this->h2 or it will delete it.
      */
      if (h->ha_index_end())
      {
        h2=new_h2;
        goto error;
      }

      h2= new_h2; /* Ok, now can put it into h2 */
      table->prepare_for_position();
      h2->extra(HA_EXTRA_KEYREAD);
    
      if (h2->ha_index_init(mrr_keyno, FALSE))
        goto error;

      use_default_impl= FALSE;
      if (pushed_cond)
        h2->idx_cond_push(mrr_keyno, pushed_cond);
    }
    else
    {
      /* 
        We get here when the access alternates betwen MRR scan(s) and non-MRR
        scans.

        Calling h->index_end() will invoke dsmrr_close() for this object,
        which will delete h2. We need to keep it, so save put it away and dont
        let it be deleted:
      */
      handler *save_h2= h2;
      h2= NULL;
      int res= (h->inited == handler::INDEX && h->ha_index_end());
      h2= save_h2;
      use_default_impl= FALSE;
      if (res)
        goto error;
    }
  }
  else
  {
    //doing DS-MRR/CPK
    // fill-buffer-analog
    // eof
    h->mrr_iter= seq_funcs->init(seq_init_param, n_ranges, mode);
    h->mrr_funcs= *seq_funcs;
    dsmrr_fill_buffer_cpk();
    if (dsmrr_eof) 
      buf->end_of_used_area= rowids_buf_last;
    DBUG_RETURN(0); // nothing can go wrong while filling the buffer
  }

  if (h2->handler::multi_range_read_init(seq_funcs, seq_init_param, n_ranges,
                                         key_parts, mode, buf) || 
      dsmrr_fill_buffer())
  {
    goto error;
  }
  /*
    If the above call has scanned through all intervals in *seq, then
    adjust *buf to indicate that the remaining buffer space will not be used.
  */
  if (dsmrr_eof) 
    buf->end_of_used_area= rowids_buf_last;

  /*
     h->inited == INDEX may occur when 'range checked for each record' is
     used.
  */
  if ((h->inited != handler::RND) && 
      ((h->inited==handler::INDEX? h->ha_index_end(): FALSE) || 
       (h->ha_rnd_init(FALSE))))
      goto error;

  use_default_impl= FALSE;
  h->mrr_funcs= *seq_funcs;
  
  DBUG_RETURN(0);
error:
  h2->ha_index_or_rnd_end();
  h2->ha_external_lock(current_thd, F_UNLCK);
  h2->close();
  delete h2;
  h2= NULL;
  DBUG_RETURN(1);
}


void DsMrr_impl::dsmrr_close()
{
  DBUG_ENTER("DsMrr_impl::dsmrr_close");
  if (h2)
  {
    h2->ha_index_or_rnd_end();
    h2->ha_external_lock(current_thd, F_UNLCK);
    h2->close();
    delete h2;
    h2= NULL;
  }
  use_default_impl= TRUE;
  DBUG_VOID_RETURN;
}


static int rowid_cmp(void *h, uchar *a, uchar *b)
{
  return ((handler*)h)->cmp_ref(a, b);
}


/**
  DS-MRR: Fill the buffer with rowids and sort it by rowid

  {This is an internal function of DiskSweep MRR implementation}
  Scan the MRR ranges and collect ROWIDs (or {ROWID, range_id} pairs) into 
  buffer. When the buffer is full or scan is completed, sort the buffer by 
  rowid and return.
  
  The function assumes that rowids buffer is empty when it is invoked. 
  
  @param h  Table handler

  @retval 0      OK, the next portion of rowids is in the buffer,
                 properly ordered
  @retval other  Error
*/

int DsMrr_impl::dsmrr_fill_buffer()
{
  char *range_info;
  int res;
  DBUG_ENTER("DsMrr_impl::dsmrr_fill_buffer");

  rowids_buf_cur= rowids_buf;
  while ((rowids_buf_cur < rowids_buf_end) && 
         !(res= h2->handler::multi_range_read_next(&range_info)))
  {
    KEY_MULTI_RANGE *curr_range= &h2->handler::mrr_cur_range;
    if (h2->mrr_funcs.skip_index_tuple &&
        h2->mrr_funcs.skip_index_tuple(h2->mrr_iter, curr_range->ptr))
      continue;
    
    /* Put rowid, or {rowid, range_id} pair into the buffer */
    h2->position(table->record[0]);
    memcpy(rowids_buf_cur, h2->ref, h2->ref_length);
    rowids_buf_cur += h2->ref_length;

    if (is_mrr_assoc)
    {
      memcpy(rowids_buf_cur, &range_info, sizeof(void*));
      rowids_buf_cur += sizeof(void*);
    }
  }

  if (res && res != HA_ERR_END_OF_FILE)
    DBUG_RETURN(res); 
  dsmrr_eof= test(res == HA_ERR_END_OF_FILE);

  /* Sort the buffer contents by rowid */
  uint elem_size= h->ref_length + (int)is_mrr_assoc * sizeof(void*);
  uint n_rowids= (rowids_buf_cur - rowids_buf) / elem_size;
  
  my_qsort2(rowids_buf, n_rowids, elem_size, (qsort2_cmp)rowid_cmp,
            (void*)h);
  rowids_buf_last= rowids_buf_cur;
  rowids_buf_cur=  rowids_buf;
  DBUG_RETURN(0);
}


/* qsort-compatible function to compare key tuples */
int DsMrr_impl::key_tuple_cmp(void* arg, uchar* key1, uchar* key2)
{
  DsMrr_impl *dsmrr= (DsMrr_impl*)arg;
  TABLE *table= dsmrr->h->table;
  
  KEY_PART_INFO *part= table->key_info[table->s->primary_key].key_part;
  KEY_PART_INFO *part_end= part + dsmrr->cpk_n_parts;

  //uint32 *lengths=item->field_lengths;
  for (; part < part_end; ++part)
  {
    Field* f = part->field;
    int len = part->store_length;
    int res = f->cmp(key1, key2);
    if (res)
      return res;
    key1 += len;
    key2 += len;
  }
  return 0;
}


//psergey2:
int DsMrr_impl::dsmrr_fill_buffer_cpk()
{
  int res;
  KEY_MULTI_RANGE cur_range;
  DBUG_ENTER("DsMrr_impl::dsmrr_fill_buffer_cpk");

  rowids_buf_cur= rowids_buf;
  while ((rowids_buf_cur < rowids_buf_end) && 
         !(res= h->mrr_funcs.next(h->mrr_iter, &cur_range)))
  {
    DBUG_ASSERT(cur_range.range_flag & EQ_RANGE);
    DBUG_ASSERT(cpk_tuple_length == cur_range.start_key.length);

    /* Put key, or {key, range_id} pair into the buffer */
    memcpy(rowids_buf_cur, cur_range.start_key.key, cpk_tuple_length);
    rowids_buf_cur += cpk_tuple_length;

    if (is_mrr_assoc)
    {
      memcpy(rowids_buf_cur, &cur_range.ptr, sizeof(void*));
      rowids_buf_cur += sizeof(void*);
    }
  }

  dsmrr_eof= test(res);

  /* Sort the buffer contents by rowid */
  uint elem_size= cpk_tuple_length + (int)is_mrr_assoc * sizeof(void*);
  uint n_rowids= (rowids_buf_cur - rowids_buf) / elem_size;
  
  my_qsort2(rowids_buf, n_rowids, elem_size, 
            (qsort2_cmp)DsMrr_impl::key_tuple_cmp, (void*)this);
  rowids_buf_last= rowids_buf_cur;
  rowids_buf_cur=  rowids_buf;
  DBUG_RETURN(0);
}


/*
  CPK: so, the source is 
   - buffer exhaustion/re-fill
   - advance to next range on "record-not-found" error.
   - if scanning on a prefix, enumerate all records for a key.
*/
int DsMrr_impl::dsmrr_next_cpk(char **range_info)
{
  int res;

  if (cpk_have_range)
  {
    res= h->index_next_same(table->record[0], rowids_buf_cur, cpk_tuple_length);
    if (res != HA_ERR_END_OF_FILE)
    {
      // todo
      if (is_mrr_assoc)
        memcpy(range_info, &cpk_saved_range_info, sizeof(void*));
      return res;
    }
    /* 
      Ok, we got EOF for records in this range. Fall through to get to another
      range.
    */
  }

  do
  {
    /* First, make sure we have a range at start of the buffer*/
    if (rowids_buf_cur == rowids_buf_last)
    {
      if (dsmrr_eof)
      {
        res= HA_ERR_END_OF_FILE;
        goto end;
      }
      // TODO: the return values are mix of HA_ERR_ codes and TRUE as "generic
      //       failure" error. Is this ok?
      if ((res= dsmrr_fill_buffer_cpk()))
        goto end;
    }
   
    if (rowids_buf_cur == rowids_buf_last)
    {
      res= HA_ERR_END_OF_FILE;
      goto end;
    }
    
    //TODO: skip-record calls here?
    //if (h2->mrr_funcs.skip_record &&
    //	h2->mrr_funcs.skip_record(h2->mrr_iter, (char *) cur_range_info, rowid))
    //  continue;
    
    /* Ok, got the range. Try making a lookup.  */
    uchar *lookup_tuple= rowids_buf_cur;
    rowids_buf_cur += cpk_tuple_length;
    if (is_mrr_assoc)
    {
      memcpy(cpk_saved_range_info, rowids_buf_cur, sizeof(void*));
      rowids_buf_cur += sizeof(void*) * test(is_mrr_assoc);
    }
      
    res= h->index_read(table->record[0], lookup_tuple, cpk_tuple_length, 
                       HA_READ_KEY_EXACT);

    if (res && res != HA_ERR_END_OF_FILE)
      goto end;

    if (!res)
    {
      memcpy(range_info, cpk_saved_range_info, sizeof(void*));
      cpk_have_range= !cpk_is_unique_scan;
      break;
    }
  } while (true);
 
end:
  return res;
}<commit_msg>MWL#121: DS-MRR support for clustered primary keys - Add testcases<commit_after>#include "mysql_priv.h"
#include "sql_select.h"

/****************************************************************************
 * Default MRR implementation (MRR to non-MRR converter)
 ***************************************************************************/

/**
  Get cost and other information about MRR scan over a known list of ranges

  Calculate estimated cost and other information about an MRR scan for given
  sequence of ranges.

  @param keyno           Index number
  @param seq             Range sequence to be traversed
  @param seq_init_param  First parameter for seq->init()
  @param n_ranges_arg    Number of ranges in the sequence, or 0 if the caller
                         can't efficiently determine it
  @param bufsz    INOUT  IN:  Size of the buffer available for use
                         OUT: Size of the buffer that is expected to be actually
                              used, or 0 if buffer is not needed.
  @param flags    INOUT  A combination of HA_MRR_* flags
  @param cost     OUT    Estimated cost of MRR access

  @note
    This method (or an overriding one in a derived class) must check for
    thd->killed and return HA_POS_ERROR if it is not zero. This is required
    for a user to be able to interrupt the calculation by killing the
    connection/query.

  @retval
    HA_POS_ERROR  Error or the engine is unable to perform the requested
                  scan. Values of OUT parameters are undefined.
  @retval
    other         OK, *cost contains cost of the scan, *bufsz and *flags
                  contain scan parameters.
*/

ha_rows 
handler::multi_range_read_info_const(uint keyno, RANGE_SEQ_IF *seq,
                                     void *seq_init_param, uint n_ranges_arg,
                                     uint *bufsz, uint *flags, COST_VECT *cost)
{
  KEY_MULTI_RANGE range;
  range_seq_t seq_it;
  ha_rows rows, total_rows= 0;
  uint n_ranges=0;
  THD *thd= current_thd;
  
  /* Default MRR implementation doesn't need buffer */
  *bufsz= 0;

  seq_it= seq->init(seq_init_param, n_ranges, *flags);
  while (!seq->next(seq_it, &range))
  {
    if (unlikely(thd->killed != 0))
      return HA_POS_ERROR;
    
    n_ranges++;
    key_range *min_endp, *max_endp;
    if (range.range_flag & GEOM_FLAG)
    {
      /* In this case tmp_min_flag contains the handler-read-function */
      range.start_key.flag= (ha_rkey_function) (range.range_flag ^ GEOM_FLAG);
      min_endp= &range.start_key;
      max_endp= NULL;
    }
    else
    {
      min_endp= range.start_key.length? &range.start_key : NULL;
      max_endp= range.end_key.length? &range.end_key : NULL;
    }
    if ((range.range_flag & UNIQUE_RANGE) && !(range.range_flag & NULL_RANGE))
      rows= 1; /* there can be at most one row */
    else
    {
      if (HA_POS_ERROR == (rows= this->records_in_range(keyno, min_endp, 
                                                        max_endp)))
      {
        /* Can't scan one range => can't do MRR scan at all */
        total_rows= HA_POS_ERROR;
        break;
      }
    }
    total_rows += rows;
  }
  
  if (total_rows != HA_POS_ERROR)
  {
    /* The following calculation is the same as in multi_range_read_info(): */
    *flags |= HA_MRR_USE_DEFAULT_IMPL;
    cost->zero();
    cost->avg_io_cost= 1; /* assume random seeks */
    if ((*flags & HA_MRR_INDEX_ONLY) && total_rows > 2)
      cost->io_count= index_only_read_time(keyno, (uint)total_rows);
    else
      cost->io_count= read_time(keyno, n_ranges, total_rows);
    cost->cpu_cost= (double) total_rows / TIME_FOR_COMPARE + 0.01;
  }
  return total_rows;
}


/**
  Get cost and other information about MRR scan over some sequence of ranges

  Calculate estimated cost and other information about an MRR scan for some
  sequence of ranges.

  The ranges themselves will be known only at execution phase. When this
  function is called we only know number of ranges and a (rough) E(#records)
  within those ranges.

  Currently this function is only called for "n-keypart singlepoint" ranges,
  i.e. each range is "keypart1=someconst1 AND ... AND keypartN=someconstN"

  The flags parameter is a combination of those flags: HA_MRR_SORTED,
  HA_MRR_INDEX_ONLY, HA_MRR_NO_ASSOCIATION, HA_MRR_LIMITS.

  @param keyno           Index number
  @param n_ranges        Estimated number of ranges (i.e. intervals) in the
                         range sequence.
  @param n_rows          Estimated total number of records contained within all
                         of the ranges
  @param bufsz    INOUT  IN:  Size of the buffer available for use
                         OUT: Size of the buffer that will be actually used, or
                              0 if buffer is not needed.
  @param flags    INOUT  A combination of HA_MRR_* flags
  @param cost     OUT    Estimated cost of MRR access

  @retval
    0     OK, *cost contains cost of the scan, *bufsz and *flags contain scan
          parameters.
  @retval
    other Error or can't perform the requested scan
*/

ha_rows handler::multi_range_read_info(uint keyno, uint n_ranges, uint n_rows,
                                       uint key_parts, uint *bufsz, 
                                       uint *flags, COST_VECT *cost)
{
  *bufsz= 0; /* Default implementation doesn't need a buffer */
  //psergey2-todo: assert for singlepoint ranges here?
  *flags |= HA_MRR_USE_DEFAULT_IMPL;

  cost->zero();
  cost->avg_io_cost= 1; /* assume random seeks */

  /* Produce the same cost as non-MRR code does */
  if (*flags & HA_MRR_INDEX_ONLY)
    cost->io_count= index_only_read_time(keyno, n_rows);
  else
    cost->io_count= read_time(keyno, n_ranges, n_rows);
  return 0;
}


/**
  Initialize the MRR scan

  Initialize the MRR scan. This function may do heavyweight scan 
  initialization like row prefetching/sorting/etc (NOTE: but better not do
  it here as we may not need it, e.g. if we never satisfy WHERE clause on
  previous tables. For many implementations it would be natural to do such
  initializations in the first multi_read_range_next() call)

  mode is a combination of the following flags: HA_MRR_SORTED,
  HA_MRR_INDEX_ONLY, HA_MRR_NO_ASSOCIATION 

  @param seq             Range sequence to be traversed
  @param seq_init_param  First parameter for seq->init()
  @param n_ranges        Number of ranges in the sequence
  @param mode            Flags, see the description section for the details
  @param buf             INOUT: memory buffer to be used

  @note
    One must have called index_init() before calling this function. Several
    multi_range_read_init() calls may be made in course of one query.

    Until WL#2623 is done (see its text, section 3.2), the following will 
    also hold:
    The caller will guarantee that if "seq->init == mrr_ranges_array_init"
    then seq_init_param is an array of n_ranges KEY_MULTI_RANGE structures.
    This property will only be used by NDB handler until WL#2623 is done.
     
    Buffer memory management is done according to the following scenario:
    The caller allocates the buffer and provides it to the callee by filling
    the members of HANDLER_BUFFER structure.
    The callee consumes all or some fraction of the provided buffer space, and
    sets the HANDLER_BUFFER members accordingly.
    The callee may use the buffer memory until the next multi_range_read_init()
    call is made, all records have been read, or until index_end() call is
    made, whichever comes first.

  @retval 0  OK
  @retval 1  Error
*/

int
handler::multi_range_read_init(RANGE_SEQ_IF *seq_funcs, void *seq_init_param,
                               uint n_ranges, uint key_parts, uint mode, 
                               HANDLER_BUFFER *buf)
{
  DBUG_ENTER("handler::multi_range_read_init");
  mrr_iter= seq_funcs->init(seq_init_param, n_ranges, mode);
  mrr_funcs= *seq_funcs;
  mrr_is_output_sorted= test(mode & HA_MRR_SORTED);
  mrr_have_range= FALSE;
  DBUG_RETURN(0);
}


/**
  Get next record in MRR scan

  Default MRR implementation: read the next record

  @param range_info  OUT  Undefined if HA_MRR_NO_ASSOCIATION flag is in effect
                          Otherwise, the opaque value associated with the range
                          that contains the returned record.

  @retval 0      OK
  @retval other  Error code
*/

int handler::multi_range_read_next(char **range_info)
{
  int UNINIT_VAR(result);
  int range_res;
  DBUG_ENTER("handler::multi_range_read_next");

  if (!mrr_have_range)
  {
    mrr_have_range= TRUE;
    goto start;
  }

  do
  {
    /* Save a call if there can be only one row in range. */
    if (mrr_cur_range.range_flag != (UNIQUE_RANGE | EQ_RANGE))
    {
      result= read_range_next();
      /* On success or non-EOF errors jump to the end. */
      if (result != HA_ERR_END_OF_FILE)
        break;
    }
    else
    {
      if (was_semi_consistent_read())
        goto scan_it_again;
      /*
        We need to set this for the last range only, but checking this
        condition is more expensive than just setting the result code.
      */
      result= HA_ERR_END_OF_FILE;
    }

start:
    /* Try the next range(s) until one matches a record. */
    while (!(range_res= mrr_funcs.next(mrr_iter, &mrr_cur_range)))
    {
scan_it_again:
      result= read_range_first(mrr_cur_range.start_key.keypart_map ?
                                 &mrr_cur_range.start_key : 0,
                               mrr_cur_range.end_key.keypart_map ?
                                 &mrr_cur_range.end_key : 0,
                               test(mrr_cur_range.range_flag & EQ_RANGE),
                               mrr_is_output_sorted);
      if (result != HA_ERR_END_OF_FILE)
        break;
    }
  }
  while ((result == HA_ERR_END_OF_FILE) && !range_res);

  *range_info= mrr_cur_range.ptr;
  DBUG_PRINT("exit",("handler::multi_range_read_next result %d", result));
  DBUG_RETURN(result);
}

/****************************************************************************
 * DS-MRR implementation 
 ***************************************************************************/

/**
  DS-MRR: Initialize and start MRR scan

  Initialize and start the MRR scan. Depending on the mode parameter, this
  may use default or DS-MRR implementation.

  @param h               Table handler to be used
  @param key             Index to be used
  @param seq_funcs       Interval sequence enumeration functions
  @param seq_init_param  Interval sequence enumeration parameter
  @param n_ranges        Number of ranges in the sequence.
  @param mode            HA_MRR_* modes to use
  @param buf             INOUT Buffer to use

  @retval 0     Ok, Scan started.
  @retval other Error
*/

int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs, 
                           void *seq_init_param, uint n_ranges, uint key_parts,
                           uint mode,
                           HANDLER_BUFFER *buf)
{
  uint elem_size;
  Item *pushed_cond= NULL;
  handler *new_h2= 0;
  DBUG_ENTER("DsMrr_impl::dsmrr_init");

  /*
    index_merge may invoke a scan on an object for which dsmrr_info[_const]
    has not been called, so set the owner handler here as well.
  */
  h= h_arg;
  if (mode & HA_MRR_USE_DEFAULT_IMPL || mode & HA_MRR_SORTED)
  {
    use_default_impl= TRUE;
    const int retval=
      h->handler::multi_range_read_init(seq_funcs, seq_init_param,
                                        n_ranges, key_parts, mode, buf);
    DBUG_RETURN(retval);
  }
  rowids_buf= buf->buffer;

  is_mrr_assoc= !test(mode & HA_MRR_NO_ASSOCIATION);

  if (is_mrr_assoc)
    status_var_increment(table->in_use->status_var.ha_multi_range_read_init_count);
 
  rowids_buf_end= buf->buffer_end;


  doing_cpk_scan= check_cpk_scan(h->active_index, mode); 
  if (doing_cpk_scan)
  {
    uint keylen=0;
    DBUG_ASSERT(key_parts != 0);
    //psergey2-todo: new elem_size here
    for (uint kp= 0; kp < key_parts; kp++)
      keylen += table->key_info[h->active_index].key_part[kp].store_length;

    cpk_tuple_length= keylen;
    cpk_is_unique_scan= test(table->key_info[h->active_index].key_parts == 
                             key_parts);
    cpk_have_range= FALSE;
    elem_size= keylen + (int)is_mrr_assoc * sizeof(void*);
    use_default_impl= FALSE;
  }
  else
    elem_size= h->ref_length + (int)is_mrr_assoc * sizeof(void*);

  rowids_buf_last= rowids_buf + 
                      ((rowids_buf_end - rowids_buf)/ elem_size)*
                      elem_size;
  rowids_buf_end= rowids_buf_last;

  /*
    There can be two cases:
    - This is the first call since index_init(), h2==NULL
       Need to setup h2 then.
    - This is not the first call, h2 is initalized and set up appropriately.
       The caller might have called h->index_init(), need to switch h to
       rnd_pos calls.
  */
  //psergey2-todo: don't create secondary for CPK scan.
  if (!doing_cpk_scan)
  {
    if (!h2)
    {
      /* Create a separate handler object to do rndpos() calls. */
      THD *thd= current_thd;
      /*
        ::clone() takes up a lot of stack, especially on 64 bit platforms.
        The constant 5 is an empiric result.
      */
      if (check_stack_overrun(thd, 5*STACK_MIN_SIZE, (uchar*) &new_h2))
        DBUG_RETURN(1);
      DBUG_ASSERT(h->active_index != MAX_KEY);
      uint mrr_keyno= h->active_index;

      /* Create a separate handler object to do rndpos() calls. */
      if (!(new_h2= h->clone(thd->mem_root)) || 
          new_h2->ha_external_lock(thd, F_RDLCK))
      {
        delete new_h2;
        DBUG_RETURN(1);
      }

      if (mrr_keyno == h->pushed_idx_cond_keyno)
        pushed_cond= h->pushed_idx_cond;

      /*
        Caution: this call will invoke this->dsmrr_close(). Do not put the
        created secondary table handler into this->h2 or it will delete it.
      */
      if (h->ha_index_end())
      {
        h2=new_h2;
        goto error;
      }

      h2= new_h2; /* Ok, now can put it into h2 */
      table->prepare_for_position();
      h2->extra(HA_EXTRA_KEYREAD);
    
      if (h2->ha_index_init(mrr_keyno, FALSE))
        goto error;

      use_default_impl= FALSE;
      if (pushed_cond)
        h2->idx_cond_push(mrr_keyno, pushed_cond);
    }
    else
    {
      /* 
        We get here when the access alternates betwen MRR scan(s) and non-MRR
        scans.

        Calling h->index_end() will invoke dsmrr_close() for this object,
        which will delete h2. We need to keep it, so save put it away and dont
        let it be deleted:
      */
      handler *save_h2= h2;
      h2= NULL;
      int res= (h->inited == handler::INDEX && h->ha_index_end());
      h2= save_h2;
      use_default_impl= FALSE;
      if (res)
        goto error;
    }
  }
  else
  {
    //doing DS-MRR/CPK
    // fill-buffer-analog
    // eof
    h->mrr_iter= seq_funcs->init(seq_init_param, n_ranges, mode);
    h->mrr_funcs= *seq_funcs;
    dsmrr_fill_buffer_cpk();
    if (dsmrr_eof) 
      buf->end_of_used_area= rowids_buf_last;
    DBUG_RETURN(0); // nothing can go wrong while filling the buffer
  }

  if (h2->handler::multi_range_read_init(seq_funcs, seq_init_param, n_ranges,
                                         key_parts, mode, buf) || 
      dsmrr_fill_buffer())
  {
    goto error;
  }
  /*
    If the above call has scanned through all intervals in *seq, then
    adjust *buf to indicate that the remaining buffer space will not be used.
  */
  if (dsmrr_eof) 
    buf->end_of_used_area= rowids_buf_last;

  /*
     h->inited == INDEX may occur when 'range checked for each record' is
     used.
  */
  if ((h->inited != handler::RND) && 
      ((h->inited==handler::INDEX? h->ha_index_end(): FALSE) || 
       (h->ha_rnd_init(FALSE))))
      goto error;

  use_default_impl= FALSE;
  h->mrr_funcs= *seq_funcs;
  
  DBUG_RETURN(0);
error:
  h2->ha_index_or_rnd_end();
  h2->ha_external_lock(current_thd, F_UNLCK);
  h2->close();
  delete h2;
  h2= NULL;
  DBUG_RETURN(1);
}


void DsMrr_impl::dsmrr_close()
{
  DBUG_ENTER("DsMrr_impl::dsmrr_close");
  if (h2)
  {
    h2->ha_index_or_rnd_end();
    h2->ha_external_lock(current_thd, F_UNLCK);
    h2->close();
    delete h2;
    h2= NULL;
  }
  use_default_impl= TRUE;
  DBUG_VOID_RETURN;
}


static int rowid_cmp(void *h, uchar *a, uchar *b)
{
  return ((handler*)h)->cmp_ref(a, b);
}


/**
  DS-MRR: Fill the buffer with rowids and sort it by rowid

  {This is an internal function of DiskSweep MRR implementation}
  Scan the MRR ranges and collect ROWIDs (or {ROWID, range_id} pairs) into 
  buffer. When the buffer is full or scan is completed, sort the buffer by 
  rowid and return.
  
  The function assumes that rowids buffer is empty when it is invoked. 
  
  @param h  Table handler

  @retval 0      OK, the next portion of rowids is in the buffer,
                 properly ordered
  @retval other  Error
*/

int DsMrr_impl::dsmrr_fill_buffer()
{
  char *range_info;
  int res;
  DBUG_ENTER("DsMrr_impl::dsmrr_fill_buffer");

  rowids_buf_cur= rowids_buf;
  while ((rowids_buf_cur < rowids_buf_end) && 
         !(res= h2->handler::multi_range_read_next(&range_info)))
  {
    KEY_MULTI_RANGE *curr_range= &h2->handler::mrr_cur_range;
    if (h2->mrr_funcs.skip_index_tuple &&
        h2->mrr_funcs.skip_index_tuple(h2->mrr_iter, curr_range->ptr))
      continue;
    
    /* Put rowid, or {rowid, range_id} pair into the buffer */
    h2->position(table->record[0]);
    memcpy(rowids_buf_cur, h2->ref, h2->ref_length);
    rowids_buf_cur += h2->ref_length;

    if (is_mrr_assoc)
    {
      memcpy(rowids_buf_cur, &range_info, sizeof(void*));
      rowids_buf_cur += sizeof(void*);
    }
  }

  if (res && res != HA_ERR_END_OF_FILE)
    DBUG_RETURN(res); 
  dsmrr_eof= test(res == HA_ERR_END_OF_FILE);

  /* Sort the buffer contents by rowid */
  uint elem_size= h->ref_length + (int)is_mrr_assoc * sizeof(void*);
  uint n_rowids= (rowids_buf_cur - rowids_buf) / elem_size;
  
  my_qsort2(rowids_buf, n_rowids, elem_size, (qsort2_cmp)rowid_cmp,
            (void*)h);
  rowids_buf_last= rowids_buf_cur;
  rowids_buf_cur=  rowids_buf;
  DBUG_RETURN(0);
}


/* qsort-compatible function to compare key tuples */
int DsMrr_impl::key_tuple_cmp(void* arg, uchar* key1, uchar* key2)
{
  DsMrr_impl *dsmrr= (DsMrr_impl*)arg;
  TABLE *table= dsmrr->h->table;
  
  KEY_PART_INFO *part= table->key_info[table->s->primary_key].key_part;
  KEY_PART_INFO *part_end= part + dsmrr->cpk_n_parts;

  //uint32 *lengths=item->field_lengths;
  for (; part < part_end; ++part)
  {
    Field* f = part->field;
    int len = part->store_length;
    int res = f->cmp(key1, key2);
    if (res)
      return res;
    key1 += len;
    key2 += len;
  }
  return 0;
}


//psergey2:
int DsMrr_impl::dsmrr_fill_buffer_cpk()
{
  int res;
  KEY_MULTI_RANGE cur_range;
  DBUG_ENTER("DsMrr_impl::dsmrr_fill_buffer_cpk");

  rowids_buf_cur= rowids_buf;
  while ((rowids_buf_cur < rowids_buf_end) && 
         !(res= h->mrr_funcs.next(h->mrr_iter, &cur_range)))
  {
    DBUG_ASSERT(cur_range.range_flag & EQ_RANGE);
    DBUG_ASSERT(cpk_tuple_length == cur_range.start_key.length);

    /* Put key, or {key, range_id} pair into the buffer */
    memcpy(rowids_buf_cur, cur_range.start_key.key, cpk_tuple_length);
    rowids_buf_cur += cpk_tuple_length;

    if (is_mrr_assoc)
    {
      memcpy(rowids_buf_cur, &cur_range.ptr, sizeof(void*));
      rowids_buf_cur += sizeof(void*);
    }
  }

  dsmrr_eof= test(res);

  /* Sort the buffer contents by rowid */
  uint elem_size= cpk_tuple_length + (int)is_mrr_assoc * sizeof(void*);
  uint n_rowids= (rowids_buf_cur - rowids_buf) / elem_size;
  
  my_qsort2(rowids_buf, n_rowids, elem_size, 
            (qsort2_cmp)DsMrr_impl::key_tuple_cmp, (void*)this);
  rowids_buf_last= rowids_buf_cur;
  rowids_buf_cur=  rowids_buf;
  DBUG_RETURN(0);
}


/*
  CPK: so, the source is 
   - buffer exhaustion/re-fill
   - advance to next range on "record-not-found" error.
   - if scanning on a prefix, enumerate all records for a key.
*/
int DsMrr_impl::dsmrr_next_cpk(char **range_info)
{
  int res;

  if (cpk_have_range)
  {
    res= h->index_next_same(table->record[0], rowids_buf_cur, cpk_tuple_length);
    if (res != HA_ERR_END_OF_FILE)
    {
      // todo
      if (is_mrr_assoc)
        memcpy(range_info, &cpk_saved_range_info, sizeof(void*));
      return res;
    }
    /* 
      Ok, we got EOF for records in this range. Fall through to get to another
      range.
    */
  }

  do
  {
    /* First, make sure we have a range at start of the buffer*/
    if (rowids_buf_cur == rowids_buf_last)
    {
      if (dsmrr_eof)
      {
        res= HA_ERR_END_OF_FILE;
        goto end;
      }
      // TODO: the return values are mix of HA_ERR_ codes and TRUE as "generic
      //       failure" error. Is this ok?
      if ((res= dsmrr_fill_buffer_cpk()))
        goto end;
    }
   
    if (rowids_buf_cur == rowids_buf_last)
    {
      res= HA_ERR_END_OF_FILE;
      goto end;
    }
    
    //TODO: make skip_index_tuple() calls, too?
    //TODO: skip-record calls here?
    //if (h2->mrr_funcs.skip_record &&
    //	h2->mrr_funcs.skip_record(h2->mrr_iter, (char *) cur_range_info, rowid))
    //  continue;
    
    /* Ok, got the range. Try making a lookup.  */
    uchar *lookup_tuple= rowids_buf_cur;
    rowids_buf_cur += cpk_tuple_length;
    if (is_mrr_assoc)
    {
      memcpy(cpk_saved_range_info, rowids_buf_cur, sizeof(void*));
      rowids_buf_cur += sizeof(void*) * test(is_mrr_assoc);
    }
      
    res= h->index_read(table->record[0], lookup_tuple, cpk_tuple_length, 
                       HA_READ_KEY_EXACT);

    if (res && res != HA_ERR_END_OF_FILE)
      goto end;

    if (!res)
    {
      memcpy(range_info, cpk_saved_range_info, sizeof(void*));
      cpk_have_range= !cpk_is_unique_scan;
      break;
    }
  } while (true);
 
end:
  return res;
}
{'fraction_non_alphanumeric': 0.1053414840762873, 'fraction_numerical': 0.0055635689264372555, 'mean_word_length': 2.6377113350545516, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 201, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1425018', 'n_tokens_mistral': 15215, 'n_tokens_neox': 14783, 'n_words': 4087}