<commit_before>#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>
#include <ruby.h>
#include <sys/types.h>

#if defined(_WIN32)
#include <stddef.h>
#endif

#ifndef RSTRING_PTR
#define RSTRING_PTR(s) (RSTRING(s)->ptr)
#endif

#ifndef RSTRING_LEN
#define RSTRING_LEN(s) (RSTRING(s)->len)
#endif

#define DATA_GET(FROM, TYPE, NAME) \
  Data_Get_Struct(FROM, TYPE, NAME); \
  if (NAME == NULL) { \
    rb_raise(rb_eArgError, "NULL found for " # NAME " when it shouldn't be."); \
  }

#define LEN(AT, P) (P - data - lexer->AT)
#define MARK(M, P) (lexer->M = (P) - data)
#define PTR_TO(P) (data + lexer->P)

typedef void (*listener_cb)(VALUE listener, const char *at, size_t length, int line);
typedef void (*listener_error_cb)(VALUE listener, const char *at, int line);
typedef void (*listener_long_cb)(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int line);
typedef void (*listener_pystring_cb)(VALUE listener, int start_col, const char *at, size_t length, int line);

typedef struct lexer_state {
  int cs;
  int content_len;
  int line_number;
  int current_line;
  int start_col;
  size_t mark;
  size_t keyword_start;
  size_t keyword_end;
  size_t next_keyword_start;
  size_t content_start;
  size_t content_end;
  size_t field_len;
  size_t query_start;
  size_t last_newline;
  size_t final_newline;

  VALUE listener;
  VALUE rows;
  VALUE current_row;

  listener_cb store_comment_content;
  listener_cb store_tag_content;
  listener_error_cb raise_lexer_error;
  listener_long_cb store_feature_content;
  listener_long_cb store_scenario_content;
  listener_long_cb store_scenario_outline_content;
  listener_long_cb store_background_content;
  listener_long_cb store_examples_content;
  listener_long_cb store_step_content;
  listener_pystring_cb store_pystring_content;
} lexer_state;

static VALUE mGherkin;
static VALUE mLexer;
static VALUE cCLexer;
static VALUE rb_eGherkinLexerError;

/** Machine **/

%%{
  machine lexer;
 
  action begin_content {
		MARK(content_start, p);
    lexer->current_line = lexer->line_number;
  }
  
  action begin_pystring_content {
    MARK(content_start, p);
  }

  action start_pystring {
    lexer->current_line = lexer->line_number;
    lexer->start_col = p - data - lexer->last_newline;
  }
 
  action store_pystring_content {
    if (lexer->store_pystring_content != NULL) {
      int len = LEN(content_start, PTR_TO(final_newline));
      if (len >= 0) { // not an empty pystring
        lexer->store_pystring_content(lexer->listener, lexer->start_col, PTR_TO(content_start), LEN(content_start, PTR_TO(final_newline)), lexer->current_line); 
      }
      else { // Empty pystring
        lexer->store_pystring_content(lexer->listener, lexer->start_col, PTR_TO(content_start), 0, lexer->current_line); 
      } 
    }
  } 
 
  action store_feature_content {
    if (lexer->store_feature_content != NULL) {
			lexer->store_feature_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), lexer->current_line);
      if(lexer->content_end != 0) {
        p = PTR_TO(content_end - 1);
      }
      lexer->content_end = 0;
    }
  }
  
  action store_background_content {
    if (lexer->store_background_content != NULL) {
			lexer->store_background_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), lexer->current_line);
      if(lexer->content_end != 0) {
        p = PTR_TO(content_end - 1);
      }
      lexer->content_end = 0;
    }
  }
  
  action store_scenario_content {
    if (lexer->store_scenario_content != NULL) {
			lexer->store_scenario_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), lexer->current_line);
      if(lexer->content_end != 0) {
        p = PTR_TO(content_end - 1);
      }
      lexer->content_end = 0;
    }
  }
  
  action store_scenario_outline_content {
    if (lexer->store_scenario_outline_content != NULL) {
			lexer->store_scenario_outline_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), lexer->current_line);
      if(lexer->content_end != 0) {
        p = PTR_TO(content_end - 1);
      }
      lexer->content_end = 0;
    }
  }
  
  action store_examples_content {
    if (lexer->store_examples_content != NULL) {
			lexer->store_examples_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), lexer->current_line);
      if(lexer->content_end != 0) {
        p = PTR_TO(content_end - 1);
      }
      lexer->content_end = 0;
    }
  }
  
  action store_step_content {
    if (lexer->store_step_content != NULL) {
			lexer->store_step_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, p), lexer->current_line);
    }
  }
  
  action store_comment_content {
    if(lexer->store_comment_content != NULL) {
      lexer->store_comment_content(lexer->listener, PTR_TO(content_start), LEN(content_start, p), lexer->line_number);
    }
  }
  
  action store_tag_content {
    if (lexer->store_tag_content != NULL) {
      lexer->store_tag_content(lexer->listener, PTR_TO(content_start), LEN(content_start, p), lexer->line_number);
    }
  }
  
  action inc_line_number {
    lexer->line_number += 1;
    MARK(final_newline, p);
  }
    
  action last_newline {
    MARK(last_newline, p + 1);
  }
 
  action start_keyword {
    if(lexer->mark == 0) {
      MARK(mark, p);
    }
  }
  
  action end_keyword {
    MARK(keyword_end, p);
    MARK(keyword_start, PTR_TO(mark));
    MARK(content_start, p + 1);
    lexer->mark = 0;
  }
  
  action next_keyword_start {
    MARK(content_end, p);
  }

  action start_table {
    p = p - 1;
    lexer->current_line = lexer->line_number;
    rb_ary_clear(lexer->rows);
    rb_ary_clear(lexer->current_row);
  }
  
  action begin_cell_content {
		MARK(content_start, p);
  }

  action store_cell_content {
    VALUE con = Qnil;
    con = rb_str_new(PTR_TO(content_start), LEN(content_start, p));
    rb_funcall(con, rb_intern("strip!"), 0);

    rb_ary_push(lexer->current_row, con);
  }

  action start_row {
    lexer->current_row = rb_ary_new();    
  }  

  action store_row {
    rb_ary_push(lexer->rows, lexer->current_row);
  }

  action store_table {
    rb_funcall(lexer->listener, rb_intern("table"), 2, lexer->rows, INT2FIX(lexer->current_line));
  }

  action end_feature {
    if (cs < lexer_first_final) {
      if (lexer->raise_lexer_error != NULL) {
        int count = 0;
        int newstr_count = 0;
        size_t len;
        const char *buff;
        if (lexer->last_newline != 0) {
          len = LEN(last_newline, eof);
          buff = PTR_TO(last_newline);
        } else {
          len = strlen(data);
          buff = data;
        }
        char newstr[len]; 
        for(count = 0; count < len; count++) {
          if(buff[count] == 10) {
            newstr[newstr_count] = '\0'; // terminate new string at first newline found
            break;
          } else {
            if (buff[count] == '%') {
              newstr[newstr_count++] = buff[count];
              newstr[newstr_count] = buff[count];
            } else {
              newstr[newstr_count] = buff[count];
            }
          }
          newstr_count++;
        }
        int line = lexer->line_number;
        lexer_init(lexer); // Otherwise you can't scan again with the same lexer
                             // calling after raising doesn't do the trick.
        lexer->raise_lexer_error(lexer->listener, newstr, line);
      }
    }
  }

  include lexer_common "lexer_common.<%= i18n_language %>.rl";   

}%%

/** Data **/
%% write data;

static VALUE strip_i(VALUE str, VALUE ary)
{
  rb_funcall(str, rb_intern("strip!"), 0);
  rb_ary_push(ary, str);
  
  return Qnil;
}

static VALUE multiline_strip(VALUE text)
{
  VALUE map = rb_ary_new();
  VALUE split = rb_str_split(text, "\n");
  
  rb_iterate(rb_each, split, strip_i, map);
  
  return rb_ary_join(split, rb_str_new2("\n"));
}

static void store_comment_content(VALUE listener, const char *at, size_t length, int line) 
{ 
  VALUE val = Qnil;

  val = rb_str_new(at, length);
  rb_funcall(listener, rb_intern("comment"), 2, val, INT2FIX(line));
}

static void store_tag_content(VALUE listener, const char *at, size_t length, int line)
{
  VALUE val = Qnil;

  val = rb_str_new(at, length);
  rb_funcall(listener, rb_intern("tag"), 2, val, INT2FIX(line)); 
} 

static void raise_lexer_error(VALUE listener, const char *at, int line)
{ 
  rb_raise(rb_eGherkinLexerError, "Parsing error on line %d: '%s'.", line, at);
}

static void store_feature_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("chop!"), 0);
  rb_funcall(listener, rb_intern("feature"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_scenario_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  con = multiline_strip(con);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("chop!"), 0);
  rb_funcall(listener, rb_intern("scenario"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_scenario_outline_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  con = multiline_strip(con);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("chop!"), 0);
  rb_funcall(listener, rb_intern("scenario_outline"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_background_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  con = multiline_strip(con);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("chop!"), 0);
  rb_funcall(listener, rb_intern("background"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_examples_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  con = multiline_strip(con);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("chop!"), 0);
  rb_funcall(listener, rb_intern("examples"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_step_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  con = multiline_strip(con);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(listener, rb_intern("step"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_pystring_content(VALUE listener, int start_col, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil;
  con = rb_str_new(at, length);
  rb_funcall(listener, rb_intern("py_string"), 3, INT2FIX(start_col), con, INT2FIX(current_line));
}

static void CLexer_free(void *data) 
{
  if (data) {
    free(data);
  }
}

static void lexer_init(lexer_state *lexer) {
  int cs = 0;
  %% write init;
  lexer->cs = cs;
  lexer->content_start = 0;
  lexer->content_end = 0;
  lexer->content_len = 0;
  lexer->mark = 0;
  lexer->field_len = 0;
  lexer->keyword_start = 0;
  lexer->keyword_end = 0;
  lexer->next_keyword_start = 0;
  lexer->line_number = 1;
  lexer->last_newline = 0;
  lexer->final_newline = 0;
  lexer->start_col = 0;
  lexer->rows = rb_ary_new();
  lexer->current_row = rb_ary_new();
}

static VALUE CLexer_alloc(VALUE klass)
{
  VALUE obj;
  lexer_state *lxr = ALLOC(lexer_state);
  lxr->store_comment_content = store_comment_content;
  lxr->store_tag_content = store_tag_content;
  lxr->store_feature_content = store_feature_content;
  lxr->store_scenario_content = store_scenario_content;
  lxr->store_scenario_outline_content = store_scenario_outline_content;
  lxr->store_background_content = store_background_content;
  lxr->store_examples_content = store_examples_content;
  lxr->store_step_content = store_step_content;
  lxr->store_pystring_content = store_pystring_content;
  lxr->raise_lexer_error = raise_lexer_error;
  lexer_init(lxr);

  obj = Data_Wrap_Struct(klass, NULL, CLexer_free, lxr);

  return obj;
}

static VALUE CLexer_init(VALUE self, VALUE listener)
{
  rb_iv_set(self, "@listener", listener);
  
  lexer_state *lxr = NULL;
  DATA_GET(self, lexer_state, lxr);
  lexer_init(lxr);
  lxr->listener = listener;
  
  return self;
}

static VALUE CLexer_scan(VALUE self, VALUE input)
{
  lexer_state *lexer = NULL;
  DATA_GET(self, lexer_state, lexer);

  rb_str_append(input, rb_str_new2("\n%_FEATURE_END_%"));
  char *data = RSTRING_PTR(input);
  long len = RSTRING_LEN(input);
 
  if (len == 0) { 
    rb_raise(rb_eGherkinLexerError, 0, "No content to parse.");
  } else {
    const char *p, *pe, *eof;
    int cs = lexer->cs;
    
    p = data;
    pe = data + len;
    eof = pe;
    
    assert(*pe == '\0' && "pointer does not end on NULL");
    assert(pe - p == len && "pointers aren't same distance");
    
    %% write exec;
    
    assert(p <= pe && "data overflow after parsing execute");
    assert(lexer->content_start <= len && "content starts after data end");
    assert(lexer->mark < len && "mark is after data end");
    assert(lexer->field_len <= len && "field has length longer than the whole data");
    
    // Reset lexer by re-initializing the whole thing
    lexer_init(lexer);

    if (lexer->cs == lexer_error) {
      rb_raise(rb_eGherkinLexerError, 0, "Invalid format, parsing fails.");<commit_msg>And that's the end of the Segfaults<commit_after>#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>
#include <ruby.h>
#include <sys/types.h>

#if defined(_WIN32)
#include <stddef.h>
#endif

#ifndef RSTRING_PTR
#define RSTRING_PTR(s) (RSTRING(s)->ptr)
#endif

#ifndef RSTRING_LEN
#define RSTRING_LEN(s) (RSTRING(s)->len)
#endif

#define DATA_GET(FROM, TYPE, NAME) \
  Data_Get_Struct(FROM, TYPE, NAME); \
  if (NAME == NULL) { \
    rb_raise(rb_eArgError, "NULL found for " # NAME " when it shouldn't be."); \
  }

#define LEN(AT, P) (P - data - lexer->AT)
#define MARK(M, P) (lexer->M = (P) - data)
#define PTR_TO(P) (data + lexer->P)

typedef void (*listener_cb)(VALUE listener, const char *at, size_t length, int line);
typedef void (*listener_error_cb)(VALUE listener, const char *at, int line);
typedef void (*listener_long_cb)(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int line);
typedef void (*listener_pystring_cb)(VALUE listener, int start_col, const char *at, size_t length, int line);

typedef struct lexer_state {
  int cs;
  int content_len;
  int line_number;
  int current_line;
  int start_col;
  size_t mark;
  size_t keyword_start;
  size_t keyword_end;
  size_t next_keyword_start;
  size_t content_start;
  size_t content_end;
  size_t field_len;
  size_t query_start;
  size_t last_newline;
  size_t final_newline;

  VALUE listener; // Possible memory leak, but Valgrind doesn't seem to detect it

  listener_cb store_comment_content;
  listener_cb store_tag_content;
  listener_error_cb raise_lexer_error;
  listener_long_cb store_feature_content;
  listener_long_cb store_scenario_content;
  listener_long_cb store_scenario_outline_content;
  listener_long_cb store_background_content;
  listener_long_cb store_examples_content;
  listener_long_cb store_step_content;
  listener_pystring_cb store_pystring_content;
} lexer_state;

static VALUE mGherkin;
static VALUE mLexer;
static VALUE cCLexer;
static VALUE rb_eGherkinLexerError;

/** Machine **/

%%{
  machine lexer;
 
  action begin_content {
		MARK(content_start, p);
    lexer->current_line = lexer->line_number;
  }
  
  action begin_pystring_content {
    MARK(content_start, p);
  }

  action start_pystring {
    lexer->current_line = lexer->line_number;
    lexer->start_col = p - data - lexer->last_newline;
  }
 
  action store_pystring_content {
    if (lexer->store_pystring_content != NULL) {
      int len = LEN(content_start, PTR_TO(final_newline));
      if (len >= 0) { // not an empty pystring
        lexer->store_pystring_content(lexer->listener, lexer->start_col, PTR_TO(content_start), LEN(content_start, PTR_TO(final_newline)), lexer->current_line); 
      }
      else { // Empty pystring
        lexer->store_pystring_content(lexer->listener, lexer->start_col, PTR_TO(content_start), 0, lexer->current_line); 
      } 
    }
  } 
 
  action store_feature_content {
    if (lexer->store_feature_content != NULL) {
			lexer->store_feature_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), lexer->current_line);
      if(lexer->content_end != 0) {
        p = PTR_TO(content_end - 1);
      }
      lexer->content_end = 0;
    }
  }
  
  action store_background_content {
    if (lexer->store_background_content != NULL) {
			lexer->store_background_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), lexer->current_line);
      if(lexer->content_end != 0) {
        p = PTR_TO(content_end - 1);
      }
      lexer->content_end = 0;
    }
  }
  
  action store_scenario_content {
    if (lexer->store_scenario_content != NULL) {
			lexer->store_scenario_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), lexer->current_line);
      if(lexer->content_end != 0) {
        p = PTR_TO(content_end - 1);
      }
      lexer->content_end = 0;
    }
  }
  
  action store_scenario_outline_content {
    if (lexer->store_scenario_outline_content != NULL) {
			lexer->store_scenario_outline_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), lexer->current_line);
      if(lexer->content_end != 0) {
        p = PTR_TO(content_end - 1);
      }
      lexer->content_end = 0;
    }
  }
  
  action store_examples_content {
    if (lexer->store_examples_content != NULL) {
			lexer->store_examples_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, PTR_TO(content_end)), lexer->current_line);
      if(lexer->content_end != 0) {
        p = PTR_TO(content_end - 1);
      }
      lexer->content_end = 0;
    }
  }
  
  action store_step_content {
    if (lexer->store_step_content != NULL) {
			lexer->store_step_content(lexer->listener, PTR_TO(keyword_start), LEN(keyword_start, PTR_TO(keyword_end)), PTR_TO(content_start), LEN(content_start, p), lexer->current_line);
    }
  }
  
  action store_comment_content {
    if(lexer->store_comment_content != NULL) {
      lexer->store_comment_content(lexer->listener, PTR_TO(content_start), LEN(content_start, p), lexer->line_number);
    }
  }
  
  action store_tag_content {
    if (lexer->store_tag_content != NULL) {
      lexer->store_tag_content(lexer->listener, PTR_TO(content_start), LEN(content_start, p), lexer->line_number);
    }
  }
  
  action inc_line_number {
    lexer->line_number += 1;
    MARK(final_newline, p);
  }
    
  action last_newline {
    MARK(last_newline, p + 1);
  }
 
  action start_keyword {
    if(lexer->mark == 0) {
      MARK(mark, p);
    }
  }
  
  action end_keyword {
    MARK(keyword_end, p);
    MARK(keyword_start, PTR_TO(mark));
    MARK(content_start, p + 1);
    lexer->mark = 0;
  }
  
  action next_keyword_start {
    MARK(content_end, p);
  }

  action start_table {
    p = p - 1;
    lexer->current_line = lexer->line_number;
    rb_ary_clear(rows);
    rb_ary_clear(current_row);
  }
  
  action begin_cell_content {
		MARK(content_start, p);
  }

  action store_cell_content {
    VALUE con = Qnil;
    con = rb_str_new(PTR_TO(content_start), LEN(content_start, p));
    rb_funcall(con, rb_intern("strip!"), 0);

    rb_ary_push(current_row, con);
  }

  action start_row {
    current_row = rb_ary_new();
  }  

  action store_row {
    rb_ary_push(rows, current_row);
  }

  action store_table {
    rb_funcall(lexer->listener, rb_intern("table"), 2, rows, INT2FIX(lexer->current_line));
  }

  action end_feature {
    if (cs < lexer_first_final) {
      if (lexer->raise_lexer_error != NULL) {
        int count = 0;
        int newstr_count = 0;
        size_t len;
        const char *buff;
        if (lexer->last_newline != 0) {
          len = LEN(last_newline, eof);
          buff = PTR_TO(last_newline);
        } else {
          len = strlen(data);
          buff = data;
        }
        char newstr[len]; 
        for(count = 0; count < len; count++) {
          if(buff[count] == 10) {
            newstr[newstr_count] = '\0'; // terminate new string at first newline found
            break;
          } else {
            if (buff[count] == '%') {
              newstr[newstr_count++] = buff[count];
              newstr[newstr_count] = buff[count];
            } else {
              newstr[newstr_count] = buff[count];
            }
          }
          newstr_count++;
        }
        int line = lexer->line_number;
        lexer_init(lexer); // Otherwise you can't scan again with the same lexer
                             // calling after raising doesn't do the trick.
        lexer->raise_lexer_error(lexer->listener, newstr, line);
      }
    }
  }

  include lexer_common "lexer_common.<%= i18n_language %>.rl";   

}%%

/** Data **/
%% write data;

static VALUE strip_i(VALUE str, VALUE ary)
{
  rb_funcall(str, rb_intern("strip!"), 0);
  rb_ary_push(ary, str);
  
  return Qnil;
}

static VALUE multiline_strip(VALUE text)
{
  VALUE map = rb_ary_new();
  VALUE split = rb_str_split(text, "\n");
  
  rb_iterate(rb_each, split, strip_i, map);
  
  return rb_ary_join(split, rb_str_new2("\n"));
}

static void store_comment_content(VALUE listener, const char *at, size_t length, int line) 
{ 
  VALUE val = Qnil;

  val = rb_str_new(at, length);
  rb_funcall(listener, rb_intern("comment"), 2, val, INT2FIX(line));
}

static void store_tag_content(VALUE listener, const char *at, size_t length, int line)
{
  VALUE val = Qnil;

  val = rb_str_new(at, length);
  rb_funcall(listener, rb_intern("tag"), 2, val, INT2FIX(line)); 
} 

static void raise_lexer_error(VALUE listener, const char *at, int line)
{ 
  rb_raise(rb_eGherkinLexerError, "Parsing error on line %d: '%s'.", line, at);
}

static void store_feature_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("chop!"), 0);
  rb_funcall(listener, rb_intern("feature"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_scenario_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  con = multiline_strip(con);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("chop!"), 0);
  rb_funcall(listener, rb_intern("scenario"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_scenario_outline_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  con = multiline_strip(con);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("chop!"), 0);
  rb_funcall(listener, rb_intern("scenario_outline"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_background_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  con = multiline_strip(con);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("chop!"), 0);
  rb_funcall(listener, rb_intern("background"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_examples_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  con = multiline_strip(con);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("chop!"), 0);
  rb_funcall(listener, rb_intern("examples"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_step_content(VALUE listener, const char *keyword_at, size_t keyword_length, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil, kw = Qnil;
  kw = rb_str_new(keyword_at, keyword_length);
  con = rb_str_new(at, length);
  con = multiline_strip(con);
  rb_funcall(con, rb_intern("strip!"), 0);
  rb_funcall(kw, rb_intern("strip!"), 0);
  rb_funcall(listener, rb_intern("step"), 3, kw, con, INT2FIX(current_line)); 
}

static void store_pystring_content(VALUE listener, int start_col, const char *at, size_t length, int current_line)
{
  VALUE con = Qnil;
  con = rb_str_new(at, length);
  rb_funcall(listener, rb_intern("py_string"), 3, INT2FIX(start_col), con, INT2FIX(current_line));
}

static void CLexer_free(void *data) 
{
  if (data) {
    free(data);
  }
}

static void lexer_init(lexer_state *lexer) {
  int cs = 0;
  %% write init;
  lexer->cs = cs;
  lexer->content_start = 0;
  lexer->content_end = 0;
  lexer->content_len = 0;
  lexer->mark = 0;
  lexer->field_len = 0;
  lexer->keyword_start = 0;
  lexer->keyword_end = 0;
  lexer->next_keyword_start = 0;
  lexer->line_number = 1;
  lexer->last_newline = 0;
  lexer->final_newline = 0;
  lexer->start_col = 0;
}

static VALUE CLexer_alloc(VALUE klass)
{
  VALUE obj;
  lexer_state *lxr = ALLOC(lexer_state);
  lxr->store_comment_content = store_comment_content;
  lxr->store_tag_content = store_tag_content;
  lxr->store_feature_content = store_feature_content;
  lxr->store_scenario_content = store_scenario_content;
  lxr->store_scenario_outline_content = store_scenario_outline_content;
  lxr->store_background_content = store_background_content;
  lxr->store_examples_content = store_examples_content;
  lxr->store_step_content = store_step_content;
  lxr->store_pystring_content = store_pystring_content;
  lxr->raise_lexer_error = raise_lexer_error;
  lexer_init(lxr);

  obj = Data_Wrap_Struct(klass, NULL, CLexer_free, lxr);

  return obj;
}

static VALUE CLexer_init(VALUE self, VALUE listener)
{
  rb_iv_set(self, "@listener", listener);
  
  lexer_state *lxr = NULL;
  DATA_GET(self, lexer_state, lxr);
  lexer_init(lxr);
  lxr->listener = listener;
  
  return self;
}

static VALUE CLexer_scan(VALUE self, VALUE input)
{
  lexer_state *lexer = NULL;
  DATA_GET(self, lexer_state, lexer);

  rb_str_append(input, rb_str_new2("\n%_FEATURE_END_%"));
  char *data = RSTRING_PTR(input);
  long len = RSTRING_LEN(input);
 
  if (len == 0) { 
    rb_raise(rb_eGherkinLexerError, 0, "No content to parse.");
  } else {
    const char *p, *pe, *eof;
    int cs = lexer->cs;
    
    VALUE rows = rb_ary_new();
    VALUE current_row = rb_ary_new();
    
    p = data;
    pe = data + len;
    eof = pe;
    
    assert(*pe == '\0' && "pointer does not end on NULL");
    assert(pe - p == len && "pointers aren't same distance");
    
    %% write exec;
    
    assert(p <= pe && "data overflow after parsing execute");
    assert(lexer->content_start <= len && "content starts after data end");
    assert(lexer->mark < len && "mark is after data end");
    assert(lexer->field_len <= len && "field has length longer than the whole data");
    
    // Reset lexer by re-initializing the whole thing
    lexer_init(lexer);

    if (lexer->cs == lexer_error) {
      rb_raise(rb_eGherkinLexerError, 0, "Invalid format, parsing fails.");
{'fraction_non_alphanumeric': 0.12813079386482387, 'fraction_numerical': 0.006270015169391538, 'mean_word_length': 3.992595085829687, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 238, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5791027', 'n_tokens_mistral': 11416, 'n_tokens_neox': 10822, 'n_words': 2267}