<commit_before>%token_prefix TK_
%token_type {buffer *}
%extra_argument {config_t *ctx}
%name configparser

%include {
#include <assert.h>
#include <stdio.h>
#include <string.h>
#include "config.h"
#include "configfile.h"
#include "buffer.h"
#include "array.h"

static void configparser_push(config_t *ctx, data_config *dc, int isnew) {
  if (isnew) {
    dc->context_ndx = ctx->all_configs->used;
    assert(dc->context_ndx > ctx->current->context_ndx);
    array_insert_unique(ctx->all_configs, (data_unset *)dc);
    dc->parent = ctx->current;
  }
  array_insert_unique(ctx->configs_stack, (data_unset *)ctx->current);
  ctx->current = dc;
}

static data_config *configparser_pop(config_t *ctx) {
  data_config *old = ctx->current;
  ctx->current = (data_config *) array_pop(ctx->configs_stack);
  return old;
}

data_unset *configparser_get_variable(config_t *ctx, buffer *key) {
  data_unset *ds, *result;
  data_config *dc;

  result = NULL;
#if 0
  fprintf(stderr, "get var %s\n", key->ptr);
#endif
  for (dc = ctx->current; dc && !result; dc = dc->parent) {
#if 0
    fprintf(stderr, "get var on block: %s\n", dc->key->ptr);
#endif
    ds = array_get_element(dc->value, key->ptr);
    if (NULL != ds) {
      result = ds;
      break;
    }
  }
  if (NULL == result) {
    fprintf(stderr, "Undefined config variable: %s\n", key->ptr);
    ctx->ok = 0;
  }
  return result;
}

/* op1 is to be eat/return by this function, op1->key is not cared
   op2 is left untouch, unreferenced
 */
data_unset *configparser_merge_data(config_t *ctx, data_unset *op1, const data_unset *op2) {
  /* type mismatch */
  if (op1->type != op2->type) {
    if (op1->type == TYPE_STRING && op2->type == TYPE_INTEGER) {
      data_string *ds = (data_string *)op1;
      buffer_append_long(ds->value, ((data_integer*)op2)->value);
      return op1;
    }
    else if (op1->type == TYPE_INTEGER && op2->type == TYPE_STRING) {
      data_string *ds = data_string_init();
      buffer_append_long(ds->value, ((data_integer*)op1)->value);
      buffer_append_string_buffer(ds->value, ((data_string*)op2)->value);
      op1->free(op1);
      return (data_unset *)ds;
    }
    else {
      fprintf(stderr, "data type mismatch, cannot be merge\n");
      ctx->ok = 0;
      op1->free(op1);
      return NULL;
    }
  }

  switch (op1->type) {
    case TYPE_STRING:
      buffer_append_string_buffer(((data_string *)op1)->value, ((data_string *)op2)->value);
      break;
    case TYPE_INTEGER:
      ((data_integer *)op1)->value += ((data_integer *)op2)->value;
      break;
    case TYPE_ARRAY: {
      array *dst = ((data_array *)op1)->value;
      array *src = ((data_array *)op2)->value;
      data_unset *du;
      size_t i;

      for (i = 0; i < src->used; i ++) {
        du = (data_unset *)src->data[i];
        if (du) {
          array_insert_unique(dst, du->copy(du));
        }
      }
      break;
    default:
      assert(0);
      break;
    }
  }
  return op1;
}

}

%parse_failure {
  ctx->ok = 0;
}

input ::= metalines.
metalines ::= metalines metaline.
metalines ::= .
metaline ::= varline.
metaline ::= condlines EOL.
metaline ::= include.
metaline ::= EOL.

%type value {data_unset *}
%type expression {data_unset *}
%type context_rvalue {data_unset *}
%type aelement {data_unset *}
%type aelements {array *}
%type array {array *}
%type condline {data_config *}
%type condlines {data_config *}
%type cond {config_cond_t }
%token_destructor { buffer_free($$); }

varline ::= key(A) ASSIGN expression(B). {
  buffer_copy_string_buffer(B->key, A);
  if (NULL == array_get_element(ctx->current->value, B->key->ptr)) {
    array_insert_unique(ctx->current->value, B);
  } else {
    fprintf(stderr, "Duplicate config variable in conditional 1 %s: %s\n", 
            ctx->current->key->ptr, B->key->ptr);
    ctx->ok = 0;
    B->free(B);
  }
  buffer_free(A);
}

varline ::= key(A) APPEND expression(B). {
  array *vars = ctx->current->value;
  data_unset *du;

  if (NULL == (du = configparser_get_variable(ctx, A))) {
    fprintf(stderr, "Undefined config variable in conditional 1 %s: %s\n", 
            ctx->current->key->ptr, A->ptr);
    ctx->ok = 0;
  } else if (NULL != (du = array_get_element(vars, A->ptr))) {
    /* exists in current block */
    du = configparser_merge_data(ctx, du, B);
    buffer_copy_string_buffer(du->key, A);
    array_replace(vars, du);
  } else {
    du = configparser_merge_data(ctx, du->copy(du), B);
    buffer_copy_string_buffer(du->key, A);
    array_insert_unique(ctx->current->value, du);
  }
  buffer_free(A);
  A = NULL;
  B->free(B);
  B = NULL;
}

key(A) ::= LKEY(B). {
  if (strchr(B->ptr, '.') == NULL) {
    A = buffer_init_string("var.");
    buffer_append_string_buffer(A, B);
  }
  else {
    A = B;
    B = NULL;
  }
}

expression(A) ::= expression(B) PLUS value(C). {
  A = configparser_merge_data(ctx, B, C);
  B = NULL;
  C->free(C);
  C = NULL;
}

expression(A) ::= value(B). {
  A = B;
  B = NULL;
}

value(A) ::= key(B). {
  A = configparser_get_variable(ctx, B);
  if (!A) {
    /* make a dummy so it won't crash */
    A = (data_unset *)data_string_init();
  }
  buffer_free(B);
  B = NULL;
}

value(A) ::= STRING(B). {
  A = (data_unset *)data_string_init();
  buffer_copy_string_buffer(((data_string *)(A))->value, B);
  buffer_free(B);
}

value(A) ::= INTEGER(B). {
  A = (data_unset *)data_integer_init();
  ((data_integer *)(A))->value = strtol(B->ptr, NULL, 10);
  buffer_free(B);
}<commit_msg>make sure variable is copied after fetch. fix stupid testcase error.<commit_after>%token_prefix TK_
%token_type {buffer *}
%extra_argument {config_t *ctx}
%name configparser

%include {
#include <assert.h>
#include <stdio.h>
#include <string.h>
#include "config.h"
#include "configfile.h"
#include "buffer.h"
#include "array.h"

static void configparser_push(config_t *ctx, data_config *dc, int isnew) {
  if (isnew) {
    dc->context_ndx = ctx->all_configs->used;
    assert(dc->context_ndx > ctx->current->context_ndx);
    array_insert_unique(ctx->all_configs, (data_unset *)dc);
    dc->parent = ctx->current;
  }
  array_insert_unique(ctx->configs_stack, (data_unset *)ctx->current);
  ctx->current = dc;
}

static data_config *configparser_pop(config_t *ctx) {
  data_config *old = ctx->current;
  ctx->current = (data_config *) array_pop(ctx->configs_stack);
  return old;
}

static const data_unset *configparser_get_variable(config_t *ctx, buffer *key) {
  data_unset *ds, *result;
  data_config *dc;

  result = NULL;
#if 0
  fprintf(stderr, "get var %s\n", key->ptr);
#endif
  for (dc = ctx->current; dc && !result; dc = dc->parent) {
#if 0
    fprintf(stderr, "get var on block: %s\n", dc->key->ptr);
    array_print(dc->value, 0);
#endif
    ds = array_get_element(dc->value, key->ptr);
    if (NULL != ds) {
      result = ds;
      break;
    }
  }
  if (NULL == result) {
    fprintf(stderr, "Undefined config variable: %s\n", key->ptr);
    ctx->ok = 0;
  }
  return result;
}

/* op1 is to be eat/return by this function, op1->key is not cared
   op2 is left untouch, unreferenced
 */
data_unset *configparser_merge_data(config_t *ctx, data_unset *op1, const data_unset *op2) {
  /* type mismatch */
  if (op1->type != op2->type) {
    if (op1->type == TYPE_STRING && op2->type == TYPE_INTEGER) {
      data_string *ds = (data_string *)op1;
      buffer_append_long(ds->value, ((data_integer*)op2)->value);
      return op1;
    }
    else if (op1->type == TYPE_INTEGER && op2->type == TYPE_STRING) {
      data_string *ds = data_string_init();
      buffer_append_long(ds->value, ((data_integer*)op1)->value);
      buffer_append_string_buffer(ds->value, ((data_string*)op2)->value);
      op1->free(op1);
      return (data_unset *)ds;
    }
    else {
      fprintf(stderr, "data type mismatch, cannot be merge\n");
      ctx->ok = 0;
      op1->free(op1);
      return NULL;
    }
  }

  switch (op1->type) {
    case TYPE_STRING:
      buffer_append_string_buffer(((data_string *)op1)->value, ((data_string *)op2)->value);
      break;
    case TYPE_INTEGER:
      ((data_integer *)op1)->value += ((data_integer *)op2)->value;
      break;
    case TYPE_ARRAY: {
      array *dst = ((data_array *)op1)->value;
      array *src = ((data_array *)op2)->value;
      data_unset *du;
      size_t i;

      for (i = 0; i < src->used; i ++) {
        du = (data_unset *)src->data[i];
        if (du) {
          array_insert_unique(dst, du->copy(du));
        }
      }
      break;
    default:
      assert(0);
      break;
    }
  }
  return op1;
}

}

%parse_failure {
  ctx->ok = 0;
}

input ::= metalines.
metalines ::= metalines metaline.
metalines ::= .
metaline ::= varline.
metaline ::= condlines EOL.
metaline ::= include.
metaline ::= EOL.

%type value {data_unset *}
%type expression {data_unset *}
%type context_rvalue {data_unset *}
%type aelement {data_unset *}
%type aelements {array *}
%type array {array *}
%type condline {data_config *}
%type condlines {data_config *}
%type cond {config_cond_t }
%token_destructor { buffer_free($$); }

varline ::= key(A) ASSIGN expression(B). {
  buffer_copy_string_buffer(B->key, A);
  if (NULL == array_get_element(ctx->current->value, B->key->ptr)) {
    array_insert_unique(ctx->current->value, B);
  } else {
    fprintf(stderr, "Duplicate config variable in conditional 1 %s: %s\n", 
            ctx->current->key->ptr, B->key->ptr);
    ctx->ok = 0;
    B->free(B);
  }
  buffer_free(A);
}

varline ::= key(A) APPEND expression(B). {
  array *vars = ctx->current->value;
  const data_unset *var;
  data_unset *du;

  if (NULL != (du = array_get_element(vars, A->ptr))) {
    /* exists in current block */
    du = configparser_merge_data(ctx, du, B);
    buffer_copy_string_buffer(du->key, A);
    array_replace(vars, du);
  } else if (NULL != (var = configparser_get_variable(ctx, A))) {
    du = var->copy(var);
    du = configparser_merge_data(ctx, du->copy(du), B);
    buffer_copy_string_buffer(du->key, A);
    array_insert_unique(ctx->current->value, du);
  } else {
    fprintf(stderr, "Undefined config variable in conditional 1 %s: %s\n", 
            ctx->current->key->ptr, A->ptr);
    ctx->ok = 0;
  }
  buffer_free(A);
  A = NULL;
  B->free(B);
  B = NULL;
}

key(A) ::= LKEY(B). {
  if (strchr(B->ptr, '.') == NULL) {
    A = buffer_init_string("var.");
    buffer_append_string_buffer(A, B);
  }
  else {
    A = B;
    B = NULL;
  }
}

expression(A) ::= expression(B) PLUS value(C). {
  A = configparser_merge_data(ctx, B, C);
  B = NULL;
  C->free(C);
  C = NULL;
}

expression(A) ::= value(B). {
  A = B;
  B = NULL;
}

value(A) ::= key(B). {
  const data_unset *var = configparser_get_variable(ctx, B);
  if (var) {
    A = var->copy(var);
  }
  else {
    /* make a dummy so it won't crash */
    A = (data_unset *)data_string_init();
  }
  buffer_free(B);
  B = NULL;
}

value(A) ::= STRING(B). {
  A = (data_unset *)data_string_init();
  buffer_copy_string_buffer(((data_string *)(A))->value, B);
  buffer_free(B);
}

value(A) ::= INTEGER(B). {
  A = (data_unset *)data_integer_init();
  ((data_integer *)(A))->value = strtol(B->ptr, NULL, 10);
  buffer_free(B);
}
{'fraction_non_alphanumeric': 0.17310803523278806, 'fraction_numerical': 0.007460003595182455, 'mean_word_length': 3.3194875776397517, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 182, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3580052', 'n_tokens_mistral': 4261, 'n_tokens_neox': 4204, 'n_words': 924}