<commit_before>/* -*- c-basic-offset: 2 -*- */
/*
  Copyright(C) 2009-2017 Brazil
  Copyright(C) 2018-2020 <NAME> <<EMAIL>>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License version 2.1 as published by the Free Software Foundation.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include "grn_ctx_impl.h"
#include "grn_db.h"
#include "grn_load.h"
#include "grn_obj.h"
#include "grn_util.h"

void
grn_loader_save_error(grn_ctx *ctx, grn_loader *loader)
{
  loader->rc = ctx->rc;
  grn_strcpy(loader->errbuf, GRN_CTX_MSGSIZE, ctx->errbuf);
}

void
grn_loader_on_record_added(grn_ctx *ctx,
                           grn_loader *loader,
                           grn_id id)
{
  if (id == GRN_ID_NIL) {
    grn_loader_save_error(ctx, loader);
    loader->n_record_errors++;
  } else {
    loader->n_records++;
  }
  if (loader->output_ids) {
    GRN_UINT32_PUT(ctx, &(loader->ids), id);
  }
  if (loader->output_errors) {
    GRN_INT32_PUT(ctx, &(loader->return_codes), ctx->rc);
    grn_vector_add_element(ctx,
                           &(loader->error_messages),
                           ctx->errbuf,
                           strlen(ctx->errbuf),
                           0,
                           GRN_DB_TEXT);
  }
  ERRCLR(ctx);
}

void
grn_loader_on_column_set(grn_ctx *ctx,
                         grn_loader *loader,
                         grn_obj *column,
                         grn_id id,
                         grn_obj *key,
                         grn_obj *value)
{
  if (ctx->rc == GRN_SUCCESS) {
    return;
  }

  grn_loader_save_error(ctx, loader);

  grn_obj key_inspected;
  grn_obj value_inspected;
  grn_obj column_name;
  GRN_TEXT_INIT(&key_inspected, 0);
  GRN_TEXT_INIT(&value_inspected, 0);
  GRN_TEXT_INIT(&column_name, 0);
  if (key) {
    grn_inspect_limited(ctx, &key_inspected, key);
  } else {
    grn_obj *table;
    if (grn_obj_is_id_accessor(ctx, column) ||
        grn_obj_is_key_accessor(ctx, column)) {
      table = ((grn_accessor *)column)->obj;
    } else {
      table = grn_ctx_at(ctx, column->header.domain);
    }
    if (grn_obj_is_lexicon(ctx, table)) {
      grn_obj key_value;
      GRN_OBJ_INIT(&key_value, GRN_BULK, 0, table->header.domain);
      grn_table_get_key2(ctx, table, id, &key_value);
      grn_inspect_limited(ctx, &key_inspected, &key_value);
      GRN_OBJ_FIN(ctx, &key_value);
    }
  }
  grn_inspect_limited(ctx, &value_inspected, value);
  if (grn_obj_is_accessor(ctx, column)) {
    grn_accessor_name(ctx, column, &column_name);
  } else {
    char raw_column_name[GRN_TABLE_MAX_KEY_SIZE];
    unsigned int raw_column_name_size;
    raw_column_name_size = grn_obj_name(ctx,
                                        column,
                                        raw_column_name,
                                        GRN_TABLE_MAX_KEY_SIZE);
    GRN_TEXT_SET(ctx, &column_name, raw_column_name, raw_column_name_size);
  }
  if (GRN_TEXT_LEN(&key_inspected) > 0) {
    GRN_LOG(ctx, GRN_LOG_ERROR,
            "[table][load] failed to set column value: %s: "
            "key: <%.*s>, column: <%.*s>, value: <%.*s>",
            ctx->errbuf,
            (int)GRN_TEXT_LEN(&key_inspected),
            GRN_TEXT_VALUE(&key_inspected),
            (int)GRN_TEXT_LEN(&column_name),
            GRN_TEXT_VALUE(&column_name),
            (int)GRN_TEXT_LEN(&value_inspected),
            GRN_TEXT_VALUE(&value_inspected));
  } else {
    GRN_LOG(ctx, GRN_LOG_ERROR,
            "[table][load] failed to set column value: %s: "
            "id: <%u>: column: <%.*s>, value: <%.*s>",
            ctx->errbuf,
            id,
            (int)GRN_TEXT_LEN(&column_name),
            GRN_TEXT_VALUE(&column_name),
            (int)GRN_TEXT_LEN(&value_inspected),
            GRN_TEXT_VALUE(&value_inspected));
  }
  GRN_OBJ_FIN(ctx, &key_inspected);
  GRN_OBJ_FIN(ctx, &value_inspected);
  GRN_OBJ_FIN(ctx, &column_name);

  loader->n_column_errors++;
  ERRCLR(ctx);
}

void
grn_loader_on_no_identifier_error(grn_ctx *ctx, grn_loader *loader)
{
  char table_name[GRN_TABLE_MAX_KEY_SIZE];
  unsigned int table_name_size;
  table_name_size = grn_obj_name(ctx,
                                 loader->table,
                                 table_name,
                                 GRN_TABLE_MAX_KEY_SIZE);
  ERR(GRN_INVALID_ARGUMENT,
      "[table][load][%.*s] neither _key nor _id is assigned",
      table_name_size, table_name);
}

grn_obj *
grn_loader_get_column(grn_ctx *ctx,
                      grn_loader *loader,
                      const char *name,
                      size_t name_length)
{
  if (!loader->columns) {
    loader->columns = grn_hash_create(ctx,
                                      NULL,
                                      GRN_TABLE_MAX_KEY_SIZE,
                                      sizeof(grn_obj *),
                                      GRN_OBJ_TABLE_HASH_KEY |
                                      GRN_OBJ_KEY_VAR_SIZE |
                                      GRN_HASH_TINY);
  }

  void *value;
  grn_id id = grn_hash_get(ctx, loader->columns, name, name_length, &value);
  if (id != GRN_ID_NIL) {
    return *((grn_obj **)value);
  }

  grn_obj *column = grn_obj_column(ctx, loader->table, name, name_length);
  if (!column) {
    return NULL;
  }

  id = grn_hash_add(ctx, loader->columns, name, name_length, &value, NULL);
  memcpy(value, &column, sizeof(grn_obj *));
  grn_obj *range = grn_ctx_at(ctx, DB_OBJ(column)->range);
  if (grn_obj_is_table(ctx, range)) {
    GRN_PTR_PUT(ctx, &(loader->ranges), range);
    grn_column_get_all_index_columns(ctx, range, &(loader->indexes));
  }
  grn_column_get_all_index_columns(ctx, column, &(loader->indexes));

  return column;
}

static grn_obj *
values_add(grn_ctx *ctx, grn_loader *loader)
{<commit_msg>windows: use grn_memcpy()<commit_after>/* -*- c-basic-offset: 2 -*- */
/*
  Copyright(C) 2009-2017 Brazil
  Copyright(C) 2018-2020 <NAME> <<EMAIL>>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License version 2.1 as published by the Free Software Foundation.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include "grn_ctx_impl.h"
#include "grn_db.h"
#include "grn_load.h"
#include "grn_obj.h"
#include "grn_util.h"

void
grn_loader_save_error(grn_ctx *ctx, grn_loader *loader)
{
  loader->rc = ctx->rc;
  grn_strcpy(loader->errbuf, GRN_CTX_MSGSIZE, ctx->errbuf);
}

void
grn_loader_on_record_added(grn_ctx *ctx,
                           grn_loader *loader,
                           grn_id id)
{
  if (id == GRN_ID_NIL) {
    grn_loader_save_error(ctx, loader);
    loader->n_record_errors++;
  } else {
    loader->n_records++;
  }
  if (loader->output_ids) {
    GRN_UINT32_PUT(ctx, &(loader->ids), id);
  }
  if (loader->output_errors) {
    GRN_INT32_PUT(ctx, &(loader->return_codes), ctx->rc);
    grn_vector_add_element(ctx,
                           &(loader->error_messages),
                           ctx->errbuf,
                           strlen(ctx->errbuf),
                           0,
                           GRN_DB_TEXT);
  }
  ERRCLR(ctx);
}

void
grn_loader_on_column_set(grn_ctx *ctx,
                         grn_loader *loader,
                         grn_obj *column,
                         grn_id id,
                         grn_obj *key,
                         grn_obj *value)
{
  if (ctx->rc == GRN_SUCCESS) {
    return;
  }

  grn_loader_save_error(ctx, loader);

  grn_obj key_inspected;
  grn_obj value_inspected;
  grn_obj column_name;
  GRN_TEXT_INIT(&key_inspected, 0);
  GRN_TEXT_INIT(&value_inspected, 0);
  GRN_TEXT_INIT(&column_name, 0);
  if (key) {
    grn_inspect_limited(ctx, &key_inspected, key);
  } else {
    grn_obj *table;
    if (grn_obj_is_id_accessor(ctx, column) ||
        grn_obj_is_key_accessor(ctx, column)) {
      table = ((grn_accessor *)column)->obj;
    } else {
      table = grn_ctx_at(ctx, column->header.domain);
    }
    if (grn_obj_is_lexicon(ctx, table)) {
      grn_obj key_value;
      GRN_OBJ_INIT(&key_value, GRN_BULK, 0, table->header.domain);
      grn_table_get_key2(ctx, table, id, &key_value);
      grn_inspect_limited(ctx, &key_inspected, &key_value);
      GRN_OBJ_FIN(ctx, &key_value);
    }
  }
  grn_inspect_limited(ctx, &value_inspected, value);
  if (grn_obj_is_accessor(ctx, column)) {
    grn_accessor_name(ctx, column, &column_name);
  } else {
    char raw_column_name[GRN_TABLE_MAX_KEY_SIZE];
    unsigned int raw_column_name_size;
    raw_column_name_size = grn_obj_name(ctx,
                                        column,
                                        raw_column_name,
                                        GRN_TABLE_MAX_KEY_SIZE);
    GRN_TEXT_SET(ctx, &column_name, raw_column_name, raw_column_name_size);
  }
  if (GRN_TEXT_LEN(&key_inspected) > 0) {
    GRN_LOG(ctx, GRN_LOG_ERROR,
            "[table][load] failed to set column value: %s: "
            "key: <%.*s>, column: <%.*s>, value: <%.*s>",
            ctx->errbuf,
            (int)GRN_TEXT_LEN(&key_inspected),
            GRN_TEXT_VALUE(&key_inspected),
            (int)GRN_TEXT_LEN(&column_name),
            GRN_TEXT_VALUE(&column_name),
            (int)GRN_TEXT_LEN(&value_inspected),
            GRN_TEXT_VALUE(&value_inspected));
  } else {
    GRN_LOG(ctx, GRN_LOG_ERROR,
            "[table][load] failed to set column value: %s: "
            "id: <%u>: column: <%.*s>, value: <%.*s>",
            ctx->errbuf,
            id,
            (int)GRN_TEXT_LEN(&column_name),
            GRN_TEXT_VALUE(&column_name),
            (int)GRN_TEXT_LEN(&value_inspected),
            GRN_TEXT_VALUE(&value_inspected));
  }
  GRN_OBJ_FIN(ctx, &key_inspected);
  GRN_OBJ_FIN(ctx, &value_inspected);
  GRN_OBJ_FIN(ctx, &column_name);

  loader->n_column_errors++;
  ERRCLR(ctx);
}

void
grn_loader_on_no_identifier_error(grn_ctx *ctx, grn_loader *loader)
{
  char table_name[GRN_TABLE_MAX_KEY_SIZE];
  unsigned int table_name_size;
  table_name_size = grn_obj_name(ctx,
                                 loader->table,
                                 table_name,
                                 GRN_TABLE_MAX_KEY_SIZE);
  ERR(GRN_INVALID_ARGUMENT,
      "[table][load][%.*s] neither _key nor _id is assigned",
      table_name_size, table_name);
}

grn_obj *
grn_loader_get_column(grn_ctx *ctx,
                      grn_loader *loader,
                      const char *name,
                      size_t name_length)
{
  if (!loader->columns) {
    loader->columns = grn_hash_create(ctx,
                                      NULL,
                                      GRN_TABLE_MAX_KEY_SIZE,
                                      sizeof(grn_obj *),
                                      GRN_OBJ_TABLE_HASH_KEY |
                                      GRN_OBJ_KEY_VAR_SIZE |
                                      GRN_HASH_TINY);
  }

  void *value;
  grn_id id = grn_hash_get(ctx, loader->columns, name, name_length, &value);
  if (id != GRN_ID_NIL) {
    return *((grn_obj **)value);
  }

  grn_obj *column = grn_obj_column(ctx, loader->table, name, name_length);
  if (!column) {
    return NULL;
  }

  id = grn_hash_add(ctx, loader->columns, name, name_length, &value, NULL);
  grn_memcpy(value, &column, sizeof(grn_obj *));
  grn_obj *range = grn_ctx_at(ctx, DB_OBJ(column)->range);
  if (grn_obj_is_table(ctx, range)) {
    GRN_PTR_PUT(ctx, &(loader->ranges), range);
    grn_column_get_all_index_columns(ctx, range, &(loader->indexes));
  }
  grn_column_get_all_index_columns(ctx, column, &(loader->indexes));

  return column;
}

static grn_obj *
values_add(grn_ctx *ctx, grn_loader *loader)
{
{'fraction_non_alphanumeric': 0.11023684628834843, 'fraction_numerical': 0.0065172468605945, 'mean_word_length': 2.300026226068712, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7404942', 'n_tokens_mistral': 4429, 'n_tokens_neox': 4322, 'n_words': 840}