<commit_before>/* -*- c-basic-offset: 2 -*- */
/*
  Copyright(C) 2009-2016 Brazil

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License version 2.1 as published by the Free Software Foundation.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include "../grn_proc.h"
#include "../grn_expr.h"
#include "../grn_str.h"
#include "../grn_output.h"
#include "../grn_util.h"
#include "../grn_cache.h"

#include "../grn_ts.h"

#include <groonga/plugin.h>

#define GRN_SELECT_INTERNAL_VAR_MATCH_COLUMNS "$match_columns"

#define DEFAULT_DRILLDOWN_LIMIT           10
#define DEFAULT_DRILLDOWN_OUTPUT_COLUMNS  "_key, _nsubrecs"

grn_rc
grn_proc_syntax_expand_query(grn_ctx *ctx,
                             const char *query,
                             unsigned int query_len,
                             grn_expr_flags flags,
                             const char *query_expander_name,
                             unsigned int query_expander_name_len,
                             grn_obj *expanded_query)
{
  grn_obj *query_expander;

  query_expander = grn_ctx_get(ctx,
                               query_expander_name,
                               query_expander_name_len);
  if (!query_expander) {
    GRN_PLUGIN_ERROR(ctx,
                     GRN_INVALID_ARGUMENT,
                     "nonexistent query expansion column: <%.*s>",
                     query_expander_name_len, query_expander_name);
    return ctx->rc;
  }

  return grn_expr_syntax_expand_query(ctx, query, query_len, flags,
                                      query_expander, expanded_query);
}

static grn_expr_flags
grn_parse_query_flags(grn_ctx *ctx, const char *query_flags,
                      unsigned int query_flags_len)
{
  grn_expr_flags flags = 0;
  const char *query_flags_end = query_flags + query_flags_len;

  while (query_flags < query_flags_end) {
    if (*query_flags == '|' || *query_flags == ' ') {
      query_flags += 1;
      continue;
    }

#define CHECK_EXPR_FLAG(name)\
  if (((query_flags_end - query_flags) >= (sizeof(#name) - 1)) &&\
      (!memcmp(query_flags, #name, sizeof(#name) - 1))) {\
    flags |= GRN_EXPR_ ## name;\
    query_flags += sizeof(#name) - 1;\
    continue;\
  }

    CHECK_EXPR_FLAG(ALLOW_PRAGMA);
    CHECK_EXPR_FLAG(ALLOW_COLUMN);
    CHECK_EXPR_FLAG(ALLOW_UPDATE);
    CHECK_EXPR_FLAG(ALLOW_LEADING_NOT);

#define GRN_EXPR_NONE 0
    CHECK_EXPR_FLAG(NONE);
#undef GNR_EXPR_NONE

    GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
                     "invalid query flag: <%.*s>",
                     (int)(query_flags_end - query_flags),
                     query_flags);
    return 0;
#undef CHECK_EXPR_FLAG
  }

  return flags;
}

static int
grn_select_apply_adjuster_ensure_factor(grn_ctx *ctx, grn_obj *factor_object)
{
  if (!factor_object) {
    return 1;
  } else if (factor_object->header.domain == GRN_DB_INT32) {
    return GRN_INT32_VALUE(factor_object);
  } else {
    grn_rc rc;
    grn_obj int32_object;
    int factor;
    GRN_INT32_INIT(&int32_object, 0);
    rc = grn_obj_cast(ctx, factor_object, &int32_object, GRN_FALSE);
    if (rc == GRN_SUCCESS) {
      factor = GRN_INT32_VALUE(&int32_object);
    } else {
      /* TODO: Log or return error? */
      factor = 1;
    }
    GRN_OBJ_FIN(ctx, &int32_object);
    return factor;
  }
}

static void
grn_select_apply_adjuster_adjust(grn_ctx *ctx, grn_obj *table, grn_obj *res,
                                 grn_obj *column, grn_obj *value,
                                 grn_obj *factor)
{
  grn_obj *index;
  unsigned int n_indexes;
  int factor_value;

  n_indexes = grn_column_index(ctx, column, GRN_OP_MATCH, &index, 1, NULL);
  if (n_indexes == 0) {
    char column_name[GRN_TABLE_MAX_KEY_SIZE];
    int column_name_size;
    column_name_size = grn_obj_name(ctx, column,
                                    column_name, GRN_TABLE_MAX_KEY_SIZE);
    ERR(GRN_INVALID_ARGUMENT,
        "adjuster requires index column for the target column: <%.*s>",
        column_name_size, column_name);
    return;
  }

  factor_value = grn_select_apply_adjuster_ensure_factor(ctx, factor);

  {
    grn_search_optarg options;
    memset(&options, 0, sizeof(grn_search_optarg));

    options.mode = GRN_OP_EXACT;
    options.similarity_threshold = 0;
    options.max_interval = 0;
    options.weight_vector = NULL;
    options.vector_size = factor_value;
    options.proc = NULL;
    options.max_size = 0;
    options.scorer = NULL;

    grn_obj_search(ctx, index, value, res, GRN_OP_ADJUST, &options);
  }
}

static void
grn_select_apply_adjuster(grn_ctx *ctx, grn_obj *table, grn_obj *res,
                          grn_obj *adjuster)
{
  grn_expr *expr = (grn_expr *)adjuster;
  grn_expr_code *code, *code_end;

  code = expr->codes;
  code_end = expr->codes + expr->codes_curr;
  while (code < code_end) {
    grn_obj *column, *value, *factor;

    if (code->op == GRN_OP_PLUS) {
      code++;
      continue;
    }

    column = code->value;
    code++;
    value = code->value;
    code++;
    code++; /* op == GRN_OP_MATCH */
    if ((code_end - code) >= 2 && code[1].op == GRN_OP_STAR) {
      factor = code->value;
      code++;
      code++; /* op == GRN_OP_STAR */
    } else {
      factor = NULL;
    }
    grn_select_apply_adjuster_adjust(ctx, table, res, column, value, factor);
  }
}

void
grn_proc_select_output_columns(grn_ctx *ctx, grn_obj *res,
                               int n_hits, int offset, int limit,
                               const char *columns, int columns_len,
                               grn_obj *condition)
{
  grn_rc rc;
  grn_obj_format format;

  GRN_OBJ_FORMAT_INIT(&format, n_hits, offset, limit, offset);
  format.flags =
    GRN_OBJ_FORMAT_WITH_COLUMN_NAMES|
    GRN_OBJ_FORMAT_XML_ELEMENT_RESULTSET;
  rc = grn_output_format_set_columns(ctx, &format, res, columns, columns_len);
  if (rc != GRN_SUCCESS) {
    GRN_OBJ_FORMAT_FIN(ctx, &format);
    return;
  }

  if (format.expression) {
    grn_obj *condition_ptr;
    condition_ptr =
      grn_expr_get_or_add_var(ctx, format.expression,
                              GRN_SELECT_INTERNAL_VAR_CONDITION,
                              strlen(GRN_SELECT_INTERNAL_VAR_CONDITION));
    GRN_PTR_INIT(condition_ptr, 0, GRN_DB_OBJECT);
    GRN_PTR_SET(ctx, condition_ptr, condition);
  }
  GRN_OUTPUT_OBJ(res, &format);
  GRN_OBJ_FORMAT_FIN(ctx, &format);
}

typedef struct {
  const char *label;
  unsigned int label_len;
  const char *keys;
  unsigned int keys_len;
  const char *sortby;
  unsigned int sortby_len;
  const char *output_columns;
  unsigned int output_columns_len;
  int offset;
  int limit;
  grn_table_group_flags calc_types;
  const char *calc_target_name;
  unsigned int calc_target_name_len;
  const char *table_name;
  unsigned int table_name_len;
} drilldown_info;

static grn_table_group_flags
grn_parse_table_group_calc_types(grn_ctx *ctx,
                                 const char *calc_types,
                                 unsigned int calc_types_len)
{
  grn_table_group_flags flags = 0;
  const char *calc_types_end = calc_types + calc_types_len;

  while (calc_types < calc_types_end) {
    if (*calc_types == ',' || *calc_types == ' ') {
      calc_types += 1;
      continue;
    }

#define CHECK_TABLE_GROUP_CALC_TYPE(name)\
  if (((calc_types_end - calc_types) >= (sizeof(#name) - 1)) &&\
      (!memcmp(calc_types, #name, sizeof(#name) - 1))) {\
    flags |= GRN_TABLE_GROUP_CALC_ ## name;\
    calc_types += sizeof(#name) - 1;\
    continue;\
  }

    CHECK_TABLE_GROUP_CALC_TYPE(COUNT);
    CHECK_TABLE_GROUP_CALC_TYPE(MAX);
    CHECK_TABLE_GROUP_CALC_TYPE(MIN);
    CHECK_TABLE_GROUP_CALC_TYPE(SUM);
    CHECK_TABLE_GROUP_CALC_TYPE(AVG);

#define GRN_TABLE_GROUP_CALC_NONE 0
    CHECK_TABLE_GROUP_CALC_TYPE(NONE);
#undef GRN_TABLE_GROUP_CALC_NONE

    ERR(GRN_INVALID_ARGUMENT, "invalid table group calc type: <%.*s>",
        (int)(calc_types_end - calc_types), calc_types);
    return 0;
#undef CHECK_TABLE_GROUP_CALC_TYPE
  }

  return flags;
}

static void
drilldown_info_fill(grn_ctx *ctx,
                    drilldown_info *drilldown,
                    grn_obj *keys,
                    grn_obj *sortby,
                    grn_obj *output_columns,
                    grn_obj *offset,
                    grn_obj *limit,
                    grn_obj *calc_types,
                    grn_obj *calc_target,
                    grn_obj *table)
{
  if (keys) {
    drilldown->keys = GRN_TEXT_VALUE(keys);
    drilldown->keys_len = GRN_TEXT_LEN(keys);
  } else {
    drilldown->keys = NULL;
    drilldown->keys_len = 0;
  }

  if (sortby) {
    drilldown->sortby = GRN_TEXT_VALUE(sortby);
    drilldown->sortby_len = GRN_TEXT_LEN(sortby);
  } else {
    drilldown->sortby = NULL;
    drilldown->sortby_len = 0;
  }

  if (output_columns) {
    drilldown->output_columns = GRN_TEXT_VALUE(output_columns);
    drilldown->output_columns_len = GRN_TEXT_LEN(output_columns);
  } else {
    drilldown->output_columns = NULL;
    drilldown->output_columns_len = 0;
  }
  if (!drilldown->output_columns_len) {
    drilldown->output_columns = DEFAULT_DRILLDOWN_OUTPUT_COLUMNS;
    drilldown->output_columns_len = strlen(DEFAULT_DRILLDOWN_OUTPUT_COLUMNS);
  }

  if (offset && GRN_TEXT_LEN(offset)) {
    drilldown->offset =
      grn_atoi(GRN_TEXT_VALUE(offset), GRN_BULK_CURR(offset), NULL);
  } else {
    drilldown->offset = 0;
  }

  if (limit && GRN_TEXT_LEN(limit)) {
    drilldown->limit =
      grn_atoi(GRN_TEXT_VALUE(limit), GRN_BULK_CURR(limit), NULL);
  } else {
    drilldown->limit = DEFAULT_DRILLDOWN_LIMIT;
  }

  if (calc_types && GRN_TEXT_LEN(calc_types)) {
    drilldown->calc_types =
      grn_parse_table_group_calc_types(ctx,
                                       GRN_TEXT_VALUE(calc_types),
                                       GRN_TEXT_LEN(calc_types));
  } else {
    drilldown->calc_types = 0;
  }

  if (calc_target && GRN_TEXT_LEN(calc_target)) {
    drilldown->calc_target_name = GRN_TEXT_VALUE(calc_target);
    drilldown->calc_target_name_len = GRN_TEXT_LEN(calc_target);
  } else {
    drilldown->calc_target_name = NULL;
    drilldown->calc_target_name_len = 0;
  }

  if (table && GRN_TEXT_LEN(table)) {
    drilldown->table_name = GRN_TEXT_VALUE(table);
    drilldown->table_name_len = GRN_TEXT_LEN(table);
  } else {
    drilldown->table_name = NULL;
    drilldown->table_name_len = 0;
  }
}

static void
grn_select_drilldown(grn_ctx *ctx, grn_obj *table,
                     grn_table_sort_key *keys, uint32_t n_keys,
                     drilldown_info *drilldown)
{
  uint32_t i;
  for (i = 0; i < n_keys; i++) {
    grn_table_group_result g = {NULL, 0, 0, 1, GRN_TABLE_GROUP_CALC_COUNT, 0};
    uint32_t n_hits;
    int offset;
    int limit;

    if (drilldown->calc_target_name) {
      g.calc_target = grn_obj_column(ctx, table,
                                     drilldown->calc_target_name,
                                     drilldown->calc_target_name_len);
    }
    if (g.calc_target) {
      g.flags |= drilldown->calc_types;
    }

    grn_table_group(ctx, table, &keys[i], 1, &g, 1);
    if (ctx->rc != GRN_SUCCESS) {
      break;
    }
    n_hits = grn_table_size(ctx, g.table);

    offset = drilldown->offset;
    limit = drilldown->limit;
    grn_normalize_offset_and_limit(ctx, n_hits, &offset, &limit);

    if (drilldown->sortby_len) {
      grn_table_sort_key *sort_keys;
      uint32_t n_sort_keys;
      sort_keys = grn_table_sort_key_from_str(ctx,
                                              drilldown->sortby,
                                              drilldown->sortby_len,
                                              g.table, &n_sort_keys);
      if (sort_keys) {
        grn_obj *sorted;
        sorted = grn_table_create(ctx, NULL, 0, NULL, GRN_OBJ_TABLE_NO_KEY,
                                  NULL, g.table);
        if (sorted) {
          grn_obj_format format;
          grn_table_sort(ctx, g.table, offset, limit,
                         sorted, sort_keys, n_sort_keys);
          GRN_OBJ_FORMAT_INIT(&format, n_hits, 0, limit, offset);
          format.flags =
            GRN_OBJ_FORMAT_WITH_COLUMN_NAMES|
            GRN_OBJ_FORMAT_XML_ELEMENT_NAVIGATIONENTRY;
          grn_obj_columns(ctx, sorted,
                          drilldown->output_columns,
                          drilldown->output_columns_len,
                          &format.columns);
          GRN_OUTPUT_OBJ(sorted, &format);
          GRN_OBJ_FORMAT_FIN(ctx, &format);
          grn_obj_unlink(ctx, sorted);
        }
        grn_table_sort_key_close(ctx, sort_keys, n_sort_keys);
      }
    } else {
      grn_obj_format format;
      GRN_OBJ_FORMAT_INIT(&format, n_hits, offset, limit, offset);
      format.flags =
        GRN_OBJ_FORMAT_WITH_COLUMN_NAMES|
        GRN_OBJ_FORMAT_XML_ELEMENT_NAVIGATIONENTRY;
      grn_obj_columns(ctx, g.table,
                      drilldown->output_columns,
                      drilldown->output_columns_len,
                      &format.columns);
      GRN_OUTPUT_OBJ(g.table, &format);
      GRN_OBJ_FORMAT_FIN(ctx, &format);
    }
    grn_obj_unlink(ctx, g.table);
    GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
                  ":", "drilldown(%d)", n_hits);
  }
}

typedef enum {
  TSORT_STATUS_NOT_VISITED,
  TSORT_STATUS_VISITING,
  TSORT_STATUS_VISITED
} tsort_status;

static grn_bool
drilldown_info_visit(grn_ctx *ctx, grn_obj *labels,
                     tsort_status *statuses,
                     drilldown_info *drilldowns,
                     grn_id to, grn_obj *ids)
{
  grn_bool cycled = GRN_TRUE;

  switch (statuses[to - 1]) {
  case TSORT_STATUS_VISITING :
    cycled = GRN_TRUE;
    break;
  case TSORT_STATUS_VISITED :
    cycled = GRN_FALSE;
    break;
  case TSORT_STATUS_NOT_VISITED :
    cycled = GRN_FALSE;
    statuses[to - 1] = TSORT_STATUS_VISITING;
    {
      drilldown_info *drilldown = &(drilldowns[to - 1]);
      if (drilldown->table_name) {
        grn_id id;
        id = grn_table_get(ctx, labels,
                           drilldown->table_name, drilldown->table_name_len);
        if (id) {
          cycled = drilldown_info_visit(ctx, labels, statuses, drilldowns,
                                        id, ids);
        }
      }
    }
    if (!cycled) {
      statuses[to - 1] = TSORT_STATUS_VISITED;
      GRN_UINT32_PUT(ctx, ids, to - 1);
    }
    break;
  }

  return cycled;
}

static grn_bool
drilldown_info_tsort(grn_ctx *ctx, grn_obj *labels,
                     tsort_status *statuses,
                     drilldown_info *drilldowns, unsigned int n_drilldowns,
                     grn_obj *ids)
{
  grn_bool succeeded = GRN_TRUE;
  unsigned int i;
  for (i = 0; i < n_drilldowns; i++) {
    drilldown_info *drilldown = &(drilldowns[i]);
    grn_id id;
    id = grn_table_get(ctx, labels,
                       drilldown->label, drilldown->label_len);
    if (id) {
      if (drilldown_info_visit(ctx, labels, statuses, drilldowns, id, ids)) {
        succeeded = GRN_FALSE;
        break;
      }
    }
  }
  return succeeded;
}

static void
drilldown_info_tsort_init(grn_ctx *ctx, grn_obj *labels,
                          tsort_status *statuses,
                          drilldown_info *drilldowns, unsigned int n_drilldowns)
{
  unsigned int i;
  for (i = 0; i < n_drilldowns; i++) {
    drilldown_info *drilldown = &(drilldowns[i]);
    int added;
    grn_id id;
    id = grn_table_add(ctx, labels,
                       drilldown->label, drilldown->label_len,
                       &added);
    if (added) {
      statuses[id - 1] = TSORT_STATUS_NOT_VISITED;
    }
  }
}

static void
grn_select_drilldowns(grn_ctx *ctx, grn_obj *table,
                      drilldown_info *drilldowns, unsigned int n_drilldowns,
                      grn_obj *condition)
{
  unsigned int i;
  grn_table_group_result *results;
  grn_obj *labels = NULL;
  grn_obj tsorted_ids;

  labels = grn_table_create(ctx, NULL, 0, NULL,
                            GRN_OBJ_TABLE_HASH_KEY,
                            grn_ctx_at(ctx, GRN_DB_SHORT_TEXT),
                            NULL);
  if (!labels) {
    return;
  }

  GRN_UINT32_INIT(&tsorted_ids, GRN_OBJ_VECTOR);

  {
    tsort_status *statuses;
    grn_bool succeeded;

    statuses = GRN_PLUGIN_MALLOCN(ctx, tsort_status, n_drilldowns);
    drilldown_info_tsort_init(ctx, labels, statuses, drilldowns, n_drilldowns);
    succeeded = drilldown_info_tsort(ctx, labels, statuses,
                                     drilldowns, n_drilldowns, &tsorted_ids);
    GRN_PLUGIN_FREE(ctx, statuses);
    if (!succeeded) {
      /* cyclic */
      goto exit;
    }
  }

  results = GRN_PLUGIN_MALLOCN(ctx, grn_table_group_result, n_drilldowns);

  /* TODO: Remove invalid key drilldowns from the count. */
  for (i = 0; i < n_drilldowns; i++) {
    grn_table_sort_key *keys = NULL;
    unsigned int n_keys;
    grn_obj *target_table = table;
    unsigned int j = GRN_UINT32_VALUE_AT(&tsorted_ids, i);
    drilldown_info *drilldown = &(drilldowns[j]);

    results[j].table = NULL;
    results[j].limit = 1;
    results[j].flags = GRN_TABLE_GROUP_CALC_COUNT;
    results[j].op = 0;
    results[j].max_n_subrecs = 0;
    results[j].calc_target = NULL;

    if (drilldown->table_name) {
      grn_id id;
      id = grn_table_get(ctx, labels,
                        drilldown->table_name, drilldown->table_name_len);
      if (id) {
        target_table = results[id - 1].table;
      }
    }

    keys = grn_table_sort_key_from_str(ctx,
                                       drilldown->keys,
                                       drilldown->keys_len,
                                       target_table, &n_keys);
    if (!keys) {
      continue;
    }

    results[j].key_begin = 0;
    results[j].key_end = n_keys - 1;
    if (n_keys > 1) {
      results[j].max_n_subrecs = 1;
    }
    if (drilldown->calc_target_name) {
      results[j].calc_target = grn_obj_column(ctx, target_table,
                                              drilldown->calc_target_name,
                                              drilldown->calc_target_name_len);
    }
    if (results[j].calc_target) {
      results[j].flags |= drilldown->calc_types;
    }

    grn_table_group(ctx, target_table, keys, n_keys, &(results[j]), 1);
    grn_table_sort_key_close(ctx, keys, n_keys);
  }

  GRN_OUTPUT_MAP_OPEN("DRILLDOWNS", n_drilldowns);
  for (i = 0; i < n_drilldowns; i++) {
    drilldown_info *drilldown = &(drilldowns[i]);
    uint32_t n_hits;
    int offset;
    int limit;

    if (!results[i].table) {
      continue;
    }

    GRN_OUTPUT_STR(drilldown->label, drilldown->label_len);

    n_hits = grn_table_size(ctx, results[i].table);

    offset = drilldown->offset;
    limit = drilldown->limit;
    grn_normalize_offset_and_limit(ctx, n_hits, &offset, &limit);

    if (drilldown->sortby_len) {
      grn_table_sort_key *sort_keys;
      uint32_t n_sort_keys;
      sort_keys = grn_table_sort_key_from_str(ctx,
                                              drilldown->sortby,
                                              drilldown->sortby_len,
                                              results[i].table, &n_sort_keys);
      if (sort_keys) {
        grn_obj *sorted;
        sorted = grn_table_create(ctx, NULL, 0, NULL, GRN_OBJ_TABLE_NO_KEY,
                                  NULL, results[i].table);
        if (sorted) {
          grn_table_sort(ctx, results[i].table, offset, limit,
                         sorted, sort_keys, n_sort_keys);
          grn_proc_select_output_columns(ctx, sorted, n_hits, 0, limit,
                                         drilldown->output_columns,
                                         drilldown->output_columns_len,
                                         condition);
          grn_obj_unlink(ctx, sorted);
        }
        grn_table_sort_key_close(ctx, sort_keys, n_sort_keys);
      }
    } else {
      grn_proc_select_output_columns(ctx, results[i].table, n_hits, offset, limit,
                                     drilldown->output_columns,
                                     drilldown->output_columns_len,
                                     condition);
    }

    if (results[i].calc_target) {
      grn_obj_unlink(ctx, results[i].calc_target);
    }
    if (results[i].table) {
      grn_obj_unlink(ctx, results[i].table);
    }

    GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
                  ":", "drilldown(%d)[%.*s]", n_hits,
                  (int)(drilldown->label_len), drilldown->label);
  }
  GRN_OUTPUT_MAP_CLOSE();
  GRN_PLUGIN_FREE(ctx, results);

exit :
  GRN_OBJ_FIN(ctx, &tsorted_ids);
  grn_obj_unlink(ctx, labels);
}

static grn_rc
grn_select(grn_ctx *ctx, const char *table, unsigned int table_len,
           const char *match_columns, unsigned int match_columns_len,
           const char *query, unsigned int query_len,
           const char *filter, unsigned int filter_len,
           const char *scorer, unsigned int scorer_len,
           const char *sortby, unsigned int sortby_len,
           const char *output_columns, unsigned int output_columns_len,
           int offset, int limit,
           drilldown_info *drilldowns,
           unsigned int n_drilldowns,
           const char *cache, unsigned int cache_len,
           const char *match_escalation_threshold, unsigned int match_escalation_threshold_len,
           const char *query_expander, unsigned int query_expander_len,
           const char *query_flags, unsigned int query_flags_len,
           const char *adjuster, unsigned int adjuster_len)
{
  uint32_t nkeys, nhits;
  uint16_t cacheable = 1, taintable = 0;
  grn_table_sort_key *keys;
  grn_obj *outbuf = ctx->impl->output.buf;
  grn_content_type output_type = ctx->impl->output.type;
  grn_obj *table_, *match_columns_ = NULL, *cond = NULL, *scorer_, *res = NULL, *sorted;
  char cache_key[GRN_CACHE_MAX_KEY_SIZE];
  uint32_t cache_key_size;
  long long int threshold, original_threshold = 0;
  grn_cache *cache_obj = grn_cache_current_get(ctx);<commit_msg>Use more meaningful name<commit_after>/* -*- c-basic-offset: 2 -*- */
/*
  Copyright(C) 2009-2016 Brazil

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License version 2.1 as published by the Free Software Foundation.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include "../grn_proc.h"
#include "../grn_expr.h"
#include "../grn_str.h"
#include "../grn_output.h"
#include "../grn_util.h"
#include "../grn_cache.h"

#include "../grn_ts.h"

#include <groonga/plugin.h>

#define GRN_SELECT_INTERNAL_VAR_MATCH_COLUMNS "$match_columns"

#define DEFAULT_DRILLDOWN_LIMIT           10
#define DEFAULT_DRILLDOWN_OUTPUT_COLUMNS  "_key, _nsubrecs"

grn_rc
grn_proc_syntax_expand_query(grn_ctx *ctx,
                             const char *query,
                             unsigned int query_len,
                             grn_expr_flags flags,
                             const char *query_expander_name,
                             unsigned int query_expander_name_len,
                             grn_obj *expanded_query)
{
  grn_obj *query_expander;

  query_expander = grn_ctx_get(ctx,
                               query_expander_name,
                               query_expander_name_len);
  if (!query_expander) {
    GRN_PLUGIN_ERROR(ctx,
                     GRN_INVALID_ARGUMENT,
                     "nonexistent query expansion column: <%.*s>",
                     query_expander_name_len, query_expander_name);
    return ctx->rc;
  }

  return grn_expr_syntax_expand_query(ctx, query, query_len, flags,
                                      query_expander, expanded_query);
}

static grn_expr_flags
grn_parse_query_flags(grn_ctx *ctx, const char *query_flags,
                      unsigned int query_flags_len)
{
  grn_expr_flags flags = 0;
  const char *query_flags_end = query_flags + query_flags_len;

  while (query_flags < query_flags_end) {
    if (*query_flags == '|' || *query_flags == ' ') {
      query_flags += 1;
      continue;
    }

#define CHECK_EXPR_FLAG(name)\
  if (((query_flags_end - query_flags) >= (sizeof(#name) - 1)) &&\
      (!memcmp(query_flags, #name, sizeof(#name) - 1))) {\
    flags |= GRN_EXPR_ ## name;\
    query_flags += sizeof(#name) - 1;\
    continue;\
  }

    CHECK_EXPR_FLAG(ALLOW_PRAGMA);
    CHECK_EXPR_FLAG(ALLOW_COLUMN);
    CHECK_EXPR_FLAG(ALLOW_UPDATE);
    CHECK_EXPR_FLAG(ALLOW_LEADING_NOT);

#define GRN_EXPR_NONE 0
    CHECK_EXPR_FLAG(NONE);
#undef GNR_EXPR_NONE

    GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT,
                     "invalid query flag: <%.*s>",
                     (int)(query_flags_end - query_flags),
                     query_flags);
    return 0;
#undef CHECK_EXPR_FLAG
  }

  return flags;
}

static int
grn_select_apply_adjuster_ensure_factor(grn_ctx *ctx, grn_obj *factor_object)
{
  if (!factor_object) {
    return 1;
  } else if (factor_object->header.domain == GRN_DB_INT32) {
    return GRN_INT32_VALUE(factor_object);
  } else {
    grn_rc rc;
    grn_obj int32_object;
    int factor;
    GRN_INT32_INIT(&int32_object, 0);
    rc = grn_obj_cast(ctx, factor_object, &int32_object, GRN_FALSE);
    if (rc == GRN_SUCCESS) {
      factor = GRN_INT32_VALUE(&int32_object);
    } else {
      /* TODO: Log or return error? */
      factor = 1;
    }
    GRN_OBJ_FIN(ctx, &int32_object);
    return factor;
  }
}

static void
grn_select_apply_adjuster_adjust(grn_ctx *ctx, grn_obj *table, grn_obj *res,
                                 grn_obj *column, grn_obj *value,
                                 grn_obj *factor)
{
  grn_obj *index;
  unsigned int n_indexes;
  int factor_value;

  n_indexes = grn_column_index(ctx, column, GRN_OP_MATCH, &index, 1, NULL);
  if (n_indexes == 0) {
    char column_name[GRN_TABLE_MAX_KEY_SIZE];
    int column_name_size;
    column_name_size = grn_obj_name(ctx, column,
                                    column_name, GRN_TABLE_MAX_KEY_SIZE);
    ERR(GRN_INVALID_ARGUMENT,
        "adjuster requires index column for the target column: <%.*s>",
        column_name_size, column_name);
    return;
  }

  factor_value = grn_select_apply_adjuster_ensure_factor(ctx, factor);

  {
    grn_search_optarg options;
    memset(&options, 0, sizeof(grn_search_optarg));

    options.mode = GRN_OP_EXACT;
    options.similarity_threshold = 0;
    options.max_interval = 0;
    options.weight_vector = NULL;
    options.vector_size = factor_value;
    options.proc = NULL;
    options.max_size = 0;
    options.scorer = NULL;

    grn_obj_search(ctx, index, value, res, GRN_OP_ADJUST, &options);
  }
}

static void
grn_select_apply_adjuster(grn_ctx *ctx, grn_obj *table, grn_obj *res,
                          grn_obj *adjuster)
{
  grn_expr *expr = (grn_expr *)adjuster;
  grn_expr_code *code, *code_end;

  code = expr->codes;
  code_end = expr->codes + expr->codes_curr;
  while (code < code_end) {
    grn_obj *column, *value, *factor;

    if (code->op == GRN_OP_PLUS) {
      code++;
      continue;
    }

    column = code->value;
    code++;
    value = code->value;
    code++;
    code++; /* op == GRN_OP_MATCH */
    if ((code_end - code) >= 2 && code[1].op == GRN_OP_STAR) {
      factor = code->value;
      code++;
      code++; /* op == GRN_OP_STAR */
    } else {
      factor = NULL;
    }
    grn_select_apply_adjuster_adjust(ctx, table, res, column, value, factor);
  }
}

void
grn_proc_select_output_columns(grn_ctx *ctx, grn_obj *res,
                               int n_hits, int offset, int limit,
                               const char *columns, int columns_len,
                               grn_obj *condition)
{
  grn_rc rc;
  grn_obj_format format;

  GRN_OBJ_FORMAT_INIT(&format, n_hits, offset, limit, offset);
  format.flags =
    GRN_OBJ_FORMAT_WITH_COLUMN_NAMES|
    GRN_OBJ_FORMAT_XML_ELEMENT_RESULTSET;
  rc = grn_output_format_set_columns(ctx, &format, res, columns, columns_len);
  if (rc != GRN_SUCCESS) {
    GRN_OBJ_FORMAT_FIN(ctx, &format);
    return;
  }

  if (format.expression) {
    grn_obj *condition_ptr;
    condition_ptr =
      grn_expr_get_or_add_var(ctx, format.expression,
                              GRN_SELECT_INTERNAL_VAR_CONDITION,
                              strlen(GRN_SELECT_INTERNAL_VAR_CONDITION));
    GRN_PTR_INIT(condition_ptr, 0, GRN_DB_OBJECT);
    GRN_PTR_SET(ctx, condition_ptr, condition);
  }
  GRN_OUTPUT_OBJ(res, &format);
  GRN_OBJ_FORMAT_FIN(ctx, &format);
}

typedef struct {
  const char *label;
  unsigned int label_len;
  const char *keys;
  unsigned int keys_len;
  const char *sortby;
  unsigned int sortby_len;
  const char *output_columns;
  unsigned int output_columns_len;
  int offset;
  int limit;
  grn_table_group_flags calc_types;
  const char *calc_target_name;
  unsigned int calc_target_name_len;
  const char *table_name;
  unsigned int table_name_len;
} drilldown_info;

static grn_table_group_flags
grn_parse_table_group_calc_types(grn_ctx *ctx,
                                 const char *calc_types,
                                 unsigned int calc_types_len)
{
  grn_table_group_flags flags = 0;
  const char *calc_types_end = calc_types + calc_types_len;

  while (calc_types < calc_types_end) {
    if (*calc_types == ',' || *calc_types == ' ') {
      calc_types += 1;
      continue;
    }

#define CHECK_TABLE_GROUP_CALC_TYPE(name)\
  if (((calc_types_end - calc_types) >= (sizeof(#name) - 1)) &&\
      (!memcmp(calc_types, #name, sizeof(#name) - 1))) {\
    flags |= GRN_TABLE_GROUP_CALC_ ## name;\
    calc_types += sizeof(#name) - 1;\
    continue;\
  }

    CHECK_TABLE_GROUP_CALC_TYPE(COUNT);
    CHECK_TABLE_GROUP_CALC_TYPE(MAX);
    CHECK_TABLE_GROUP_CALC_TYPE(MIN);
    CHECK_TABLE_GROUP_CALC_TYPE(SUM);
    CHECK_TABLE_GROUP_CALC_TYPE(AVG);

#define GRN_TABLE_GROUP_CALC_NONE 0
    CHECK_TABLE_GROUP_CALC_TYPE(NONE);
#undef GRN_TABLE_GROUP_CALC_NONE

    ERR(GRN_INVALID_ARGUMENT, "invalid table group calc type: <%.*s>",
        (int)(calc_types_end - calc_types), calc_types);
    return 0;
#undef CHECK_TABLE_GROUP_CALC_TYPE
  }

  return flags;
}

static void
drilldown_info_fill(grn_ctx *ctx,
                    drilldown_info *drilldown,
                    grn_obj *keys,
                    grn_obj *sortby,
                    grn_obj *output_columns,
                    grn_obj *offset,
                    grn_obj *limit,
                    grn_obj *calc_types,
                    grn_obj *calc_target,
                    grn_obj *table)
{
  if (keys) {
    drilldown->keys = GRN_TEXT_VALUE(keys);
    drilldown->keys_len = GRN_TEXT_LEN(keys);
  } else {
    drilldown->keys = NULL;
    drilldown->keys_len = 0;
  }

  if (sortby) {
    drilldown->sortby = GRN_TEXT_VALUE(sortby);
    drilldown->sortby_len = GRN_TEXT_LEN(sortby);
  } else {
    drilldown->sortby = NULL;
    drilldown->sortby_len = 0;
  }

  if (output_columns) {
    drilldown->output_columns = GRN_TEXT_VALUE(output_columns);
    drilldown->output_columns_len = GRN_TEXT_LEN(output_columns);
  } else {
    drilldown->output_columns = NULL;
    drilldown->output_columns_len = 0;
  }
  if (!drilldown->output_columns_len) {
    drilldown->output_columns = DEFAULT_DRILLDOWN_OUTPUT_COLUMNS;
    drilldown->output_columns_len = strlen(DEFAULT_DRILLDOWN_OUTPUT_COLUMNS);
  }

  if (offset && GRN_TEXT_LEN(offset)) {
    drilldown->offset =
      grn_atoi(GRN_TEXT_VALUE(offset), GRN_BULK_CURR(offset), NULL);
  } else {
    drilldown->offset = 0;
  }

  if (limit && GRN_TEXT_LEN(limit)) {
    drilldown->limit =
      grn_atoi(GRN_TEXT_VALUE(limit), GRN_BULK_CURR(limit), NULL);
  } else {
    drilldown->limit = DEFAULT_DRILLDOWN_LIMIT;
  }

  if (calc_types && GRN_TEXT_LEN(calc_types)) {
    drilldown->calc_types =
      grn_parse_table_group_calc_types(ctx,
                                       GRN_TEXT_VALUE(calc_types),
                                       GRN_TEXT_LEN(calc_types));
  } else {
    drilldown->calc_types = 0;
  }

  if (calc_target && GRN_TEXT_LEN(calc_target)) {
    drilldown->calc_target_name = GRN_TEXT_VALUE(calc_target);
    drilldown->calc_target_name_len = GRN_TEXT_LEN(calc_target);
  } else {
    drilldown->calc_target_name = NULL;
    drilldown->calc_target_name_len = 0;
  }

  if (table && GRN_TEXT_LEN(table)) {
    drilldown->table_name = GRN_TEXT_VALUE(table);
    drilldown->table_name_len = GRN_TEXT_LEN(table);
  } else {
    drilldown->table_name = NULL;
    drilldown->table_name_len = 0;
  }
}

static void
grn_select_drilldown(grn_ctx *ctx, grn_obj *table,
                     grn_table_sort_key *keys, uint32_t n_keys,
                     drilldown_info *drilldown)
{
  uint32_t i;
  for (i = 0; i < n_keys; i++) {
    grn_table_group_result g = {NULL, 0, 0, 1, GRN_TABLE_GROUP_CALC_COUNT, 0};
    uint32_t n_hits;
    int offset;
    int limit;

    if (drilldown->calc_target_name) {
      g.calc_target = grn_obj_column(ctx, table,
                                     drilldown->calc_target_name,
                                     drilldown->calc_target_name_len);
    }
    if (g.calc_target) {
      g.flags |= drilldown->calc_types;
    }

    grn_table_group(ctx, table, &keys[i], 1, &g, 1);
    if (ctx->rc != GRN_SUCCESS) {
      break;
    }
    n_hits = grn_table_size(ctx, g.table);

    offset = drilldown->offset;
    limit = drilldown->limit;
    grn_normalize_offset_and_limit(ctx, n_hits, &offset, &limit);

    if (drilldown->sortby_len) {
      grn_table_sort_key *sort_keys;
      uint32_t n_sort_keys;
      sort_keys = grn_table_sort_key_from_str(ctx,
                                              drilldown->sortby,
                                              drilldown->sortby_len,
                                              g.table, &n_sort_keys);
      if (sort_keys) {
        grn_obj *sorted;
        sorted = grn_table_create(ctx, NULL, 0, NULL, GRN_OBJ_TABLE_NO_KEY,
                                  NULL, g.table);
        if (sorted) {
          grn_obj_format format;
          grn_table_sort(ctx, g.table, offset, limit,
                         sorted, sort_keys, n_sort_keys);
          GRN_OBJ_FORMAT_INIT(&format, n_hits, 0, limit, offset);
          format.flags =
            GRN_OBJ_FORMAT_WITH_COLUMN_NAMES|
            GRN_OBJ_FORMAT_XML_ELEMENT_NAVIGATIONENTRY;
          grn_obj_columns(ctx, sorted,
                          drilldown->output_columns,
                          drilldown->output_columns_len,
                          &format.columns);
          GRN_OUTPUT_OBJ(sorted, &format);
          GRN_OBJ_FORMAT_FIN(ctx, &format);
          grn_obj_unlink(ctx, sorted);
        }
        grn_table_sort_key_close(ctx, sort_keys, n_sort_keys);
      }
    } else {
      grn_obj_format format;
      GRN_OBJ_FORMAT_INIT(&format, n_hits, offset, limit, offset);
      format.flags =
        GRN_OBJ_FORMAT_WITH_COLUMN_NAMES|
        GRN_OBJ_FORMAT_XML_ELEMENT_NAVIGATIONENTRY;
      grn_obj_columns(ctx, g.table,
                      drilldown->output_columns,
                      drilldown->output_columns_len,
                      &format.columns);
      GRN_OUTPUT_OBJ(g.table, &format);
      GRN_OBJ_FORMAT_FIN(ctx, &format);
    }
    grn_obj_unlink(ctx, g.table);
    GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
                  ":", "drilldown(%d)", n_hits);
  }
}

typedef enum {
  TSORT_STATUS_NOT_VISITED,
  TSORT_STATUS_VISITING,
  TSORT_STATUS_VISITED
} tsort_status;

static grn_bool
drilldown_info_visit(grn_ctx *ctx, grn_obj *labels,
                     tsort_status *statuses,
                     drilldown_info *drilldowns,
                     grn_id to, grn_obj *indexes)
{
  grn_bool cycled = GRN_TRUE;

  switch (statuses[to - 1]) {
  case TSORT_STATUS_VISITING :
    cycled = GRN_TRUE;
    break;
  case TSORT_STATUS_VISITED :
    cycled = GRN_FALSE;
    break;
  case TSORT_STATUS_NOT_VISITED :
    cycled = GRN_FALSE;
    statuses[to - 1] = TSORT_STATUS_VISITING;
    {
      drilldown_info *drilldown = &(drilldowns[to - 1]);
      if (drilldown->table_name) {
        grn_id id;
        id = grn_table_get(ctx, labels,
                           drilldown->table_name, drilldown->table_name_len);
        if (id) {
          cycled = drilldown_info_visit(ctx, labels, statuses, drilldowns,
                                        id, indexes);
        }
      }
    }
    if (!cycled) {
      statuses[to - 1] = TSORT_STATUS_VISITED;
      GRN_UINT32_PUT(ctx, indexes, to - 1);
    }
    break;
  }

  return cycled;
}

static grn_bool
drilldown_info_tsort(grn_ctx *ctx, grn_obj *labels,
                     tsort_status *statuses,
                     drilldown_info *drilldowns, unsigned int n_drilldowns,
                     grn_obj *indexes)
{
  grn_bool succeeded = GRN_TRUE;
  unsigned int i;
  for (i = 0; i < n_drilldowns; i++) {
    drilldown_info *drilldown = &(drilldowns[i]);
    grn_id id;
    id = grn_table_get(ctx, labels,
                       drilldown->label, drilldown->label_len);
    if (id) {
      if (drilldown_info_visit(ctx, labels, statuses, drilldowns, id, indexes)) {
        succeeded = GRN_FALSE;
        break;
      }
    }
  }
  return succeeded;
}

static void
drilldown_info_tsort_init(grn_ctx *ctx, grn_obj *labels,
                          tsort_status *statuses,
                          drilldown_info *drilldowns, unsigned int n_drilldowns)
{
  unsigned int i;
  for (i = 0; i < n_drilldowns; i++) {
    drilldown_info *drilldown = &(drilldowns[i]);
    int added;
    grn_id id;
    id = grn_table_add(ctx, labels,
                       drilldown->label, drilldown->label_len,
                       &added);
    if (added) {
      statuses[id - 1] = TSORT_STATUS_NOT_VISITED;
    }
  }
}

static void
grn_select_drilldowns(grn_ctx *ctx, grn_obj *table,
                      drilldown_info *drilldowns, unsigned int n_drilldowns,
                      grn_obj *condition)
{
  unsigned int i;
  grn_table_group_result *results;
  grn_obj *labels = NULL;
  grn_obj tsorted_indexes;

  labels = grn_table_create(ctx, NULL, 0, NULL,
                            GRN_OBJ_TABLE_HASH_KEY,
                            grn_ctx_at(ctx, GRN_DB_SHORT_TEXT),
                            NULL);
  if (!labels) {
    return;
  }

  GRN_UINT32_INIT(&tsorted_indexes, GRN_OBJ_VECTOR);

  {
    tsort_status *statuses;
    grn_bool succeeded;

    statuses = GRN_PLUGIN_MALLOCN(ctx, tsort_status, n_drilldowns);
    drilldown_info_tsort_init(ctx, labels, statuses, drilldowns, n_drilldowns);
    succeeded = drilldown_info_tsort(ctx, labels, statuses,
                                     drilldowns, n_drilldowns, &tsorted_indexes);
    GRN_PLUGIN_FREE(ctx, statuses);
    if (!succeeded) {
      /* cyclic */
      goto exit;
    }
  }

  results = GRN_PLUGIN_MALLOCN(ctx, grn_table_group_result, n_drilldowns);

  /* TODO: Remove invalid key drilldowns from the count. */
  for (i = 0; i < n_drilldowns; i++) {
    grn_table_sort_key *keys = NULL;
    unsigned int n_keys;
    grn_obj *target_table = table;
    unsigned int j = GRN_UINT32_VALUE_AT(&tsorted_indexes, i);
    drilldown_info *drilldown = &(drilldowns[j]);

    results[j].table = NULL;
    results[j].limit = 1;
    results[j].flags = GRN_TABLE_GROUP_CALC_COUNT;
    results[j].op = 0;
    results[j].max_n_subrecs = 0;
    results[j].calc_target = NULL;

    if (drilldown->table_name) {
      grn_id id;
      id = grn_table_get(ctx, labels,
                        drilldown->table_name, drilldown->table_name_len);
      if (id) {
        target_table = results[id - 1].table;
      }
    }

    keys = grn_table_sort_key_from_str(ctx,
                                       drilldown->keys,
                                       drilldown->keys_len,
                                       target_table, &n_keys);
    if (!keys) {
      continue;
    }

    results[j].key_begin = 0;
    results[j].key_end = n_keys - 1;
    if (n_keys > 1) {
      results[j].max_n_subrecs = 1;
    }
    if (drilldown->calc_target_name) {
      results[j].calc_target = grn_obj_column(ctx, target_table,
                                              drilldown->calc_target_name,
                                              drilldown->calc_target_name_len);
    }
    if (results[j].calc_target) {
      results[j].flags |= drilldown->calc_types;
    }

    grn_table_group(ctx, target_table, keys, n_keys, &(results[j]), 1);
    grn_table_sort_key_close(ctx, keys, n_keys);
  }

  GRN_OUTPUT_MAP_OPEN("DRILLDOWNS", n_drilldowns);
  for (i = 0; i < n_drilldowns; i++) {
    drilldown_info *drilldown = &(drilldowns[i]);
    uint32_t n_hits;
    int offset;
    int limit;

    if (!results[i].table) {
      continue;
    }

    GRN_OUTPUT_STR(drilldown->label, drilldown->label_len);

    n_hits = grn_table_size(ctx, results[i].table);

    offset = drilldown->offset;
    limit = drilldown->limit;
    grn_normalize_offset_and_limit(ctx, n_hits, &offset, &limit);

    if (drilldown->sortby_len) {
      grn_table_sort_key *sort_keys;
      uint32_t n_sort_keys;
      sort_keys = grn_table_sort_key_from_str(ctx,
                                              drilldown->sortby,
                                              drilldown->sortby_len,
                                              results[i].table, &n_sort_keys);
      if (sort_keys) {
        grn_obj *sorted;
        sorted = grn_table_create(ctx, NULL, 0, NULL, GRN_OBJ_TABLE_NO_KEY,
                                  NULL, results[i].table);
        if (sorted) {
          grn_table_sort(ctx, results[i].table, offset, limit,
                         sorted, sort_keys, n_sort_keys);
          grn_proc_select_output_columns(ctx, sorted, n_hits, 0, limit,
                                         drilldown->output_columns,
                                         drilldown->output_columns_len,
                                         condition);
          grn_obj_unlink(ctx, sorted);
        }
        grn_table_sort_key_close(ctx, sort_keys, n_sort_keys);
      }
    } else {
      grn_proc_select_output_columns(ctx, results[i].table, n_hits, offset, limit,
                                     drilldown->output_columns,
                                     drilldown->output_columns_len,
                                     condition);
    }

    if (results[i].calc_target) {
      grn_obj_unlink(ctx, results[i].calc_target);
    }
    if (results[i].table) {
      grn_obj_unlink(ctx, results[i].table);
    }

    GRN_QUERY_LOG(ctx, GRN_QUERY_LOG_SIZE,
                  ":", "drilldown(%d)[%.*s]", n_hits,
                  (int)(drilldown->label_len), drilldown->label);
  }
  GRN_OUTPUT_MAP_CLOSE();
  GRN_PLUGIN_FREE(ctx, results);

exit :
  GRN_OBJ_FIN(ctx, &tsorted_indexes);
  grn_obj_unlink(ctx, labels);
}

static grn_rc
grn_select(grn_ctx *ctx, const char *table, unsigned int table_len,
           const char *match_columns, unsigned int match_columns_len,
           const char *query, unsigned int query_len,
           const char *filter, unsigned int filter_len,
           const char *scorer, unsigned int scorer_len,
           const char *sortby, unsigned int sortby_len,
           const char *output_columns, unsigned int output_columns_len,
           int offset, int limit,
           drilldown_info *drilldowns,
           unsigned int n_drilldowns,
           const char *cache, unsigned int cache_len,
           const char *match_escalation_threshold, unsigned int match_escalation_threshold_len,
           const char *query_expander, unsigned int query_expander_len,
           const char *query_flags, unsigned int query_flags_len,
           const char *adjuster, unsigned int adjuster_len)
{
  uint32_t nkeys, nhits;
  uint16_t cacheable = 1, taintable = 0;
  grn_table_sort_key *keys;
  grn_obj *outbuf = ctx->impl->output.buf;
  grn_content_type output_type = ctx->impl->output.type;
  grn_obj *table_, *match_columns_ = NULL, *cond = NULL, *scorer_, *res = NULL, *sorted;
  char cache_key[GRN_CACHE_MAX_KEY_SIZE];
  uint32_t cache_key_size;
  long long int threshold, original_threshold = 0;
  grn_cache *cache_obj = grn_cache_current_get(ctx);
{'fraction_non_alphanumeric': 0.09780461275500012, 'fraction_numerical': 0.005904681568958245, 'mean_word_length': 2.3154253753311744, 'pattern_counts': {'":': 4, '<': 29, '<?xml version=': 0, '>': 191, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 88, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1269273', 'n_tokens_mistral': 16216, 'n_tokens_neox': 15714, 'n_words': 2910}