<commit_before>require 'gherkin/core_ext/array'

module Gherkin
  module RbLexer
    class <%= @i18n.sanitized_key.capitalize %> #:nodoc:
      %%{
        machine lexer;
 
        action begin_content {
          @content_start = p
          @current_line = @line_number
        }

        action start_pystring {
          @current_line = @line_number
          @start_col = p - @last_newline
        }
        
        action begin_pystring_content {
          @content_start = p
        }

        action store_pystring_content {
          con = unindent(@start_col, data[@content_start...@next_keyword_start-1].utf8_pack("c*").sub(/(\r?\n)?( )*\Z/, ''))
          @listener.py_string(con, @current_line) 
        }
 
        action store_feature_content {
          store_keyword_content(:feature, data, p, eof) { |con| multiline_strip(con) }
          p = @next_keyword_start - 1 if @next_keyword_start          
          @next_keyword_start = nil
        }
 
        action store_background_content {
          store_keyword_content(:background, data, p, eof) { |con| multiline_strip(con) }
          p = @next_keyword_start - 1 if @next_keyword_start                    
          @next_keyword_start = nil
        }

        action store_scenario_content {
          store_keyword_content(:scenario, data, p, eof) { |con| multiline_strip(con) }
          p = @next_keyword_start - 1 if @next_keyword_start                    
          @next_keyword_start = nil
        }

        action store_scenario_outline_content {
          store_keyword_content(:scenario_outline, data, p, eof) { |con| multiline_strip(con) }
          p = @next_keyword_start - 1 if @next_keyword_start                    
          @next_keyword_start = nil
        }

        action store_examples_content {
          store_keyword_content(:examples, data, p, eof) { |con| multiline_strip(con) }
          p = @next_keyword_start - 1 if @next_keyword_start                    
          @next_keyword_start = nil
        }

        action store_step_content {
          con = data[@content_start...p].utf8_pack("c*").strip
          @listener.step(@keyword, con, @current_line)
        }
 
        action store_comment_content {
          con = data[@content_start...p].utf8_pack("c*").strip
          @listener.comment(con, @line_number)
          @keyword_start = nil
        }
 
        action store_tag_content {
          con = data[@content_start...p].utf8_pack("c*").strip
          @listener.tag(con, @current_line)
          @keyword_start = nil
        }
 
        action inc_line_number {
          @line_number += 1
        }
 
        action last_newline {
          @last_newline = p + 1
        }

        action start_keyword {
          @keyword_start ||= p
        }
 
        action end_keyword {
          @keyword = data[@keyword_start...p].utf8_pack("c*").sub(/:$/,'')
          @keyword_start = nil
        }
 
        action next_keyword_start {
          @next_keyword_start = p
        }

        action start_table {
          p = p - 1
          @rows = []
          @current_line = @line_number
        }

        action start_row {
          current_row = []
        }
 
        action begin_cell_content {
          @content_start = p
        }

        action store_cell_content {
          con = data[@content_start...p].utf8_pack("c*").strip
          current_row << con
        }

        action store_row {
          @rows << current_row
        }

        action store_table {
          if @rows.size != 0
            @listener.table(@rows, @current_line)
          end 
        }
 
        action end_feature {
          if cs < lexer_first_final
            content = current_line_content(data, p)
            raise Lexer::LexingError.new("Lexing error on line %d: '%s'." % [@line_number, content])
          end
        }

        include lexer_common "lexer_common.<%= @i18n.sanitized_key %>.rl"; 
      }%%
 
      def initialize(listener)
        @listener = listener
        %% write data;
      end
 
      def scan(data)
        data = (data + "\n%_FEATURE_END_%").unpack("c*") # Explicit EOF simplifies things considerably
        eof = pe = data.length
 
        @line_number = 1
        @last_newline = 0
 
        %% write init;
        %% write exec;
      end
 
      def multiline_strip(text)
        crlf_count = text.scan(/\r\n/).size
        lf_count = text.scan(/[^\r]\n/).size
        eol = crlf_count > lf_count ? "\r\n" : "\n"
        text.split(/\r?\n/).map{|s| s.strip}.join(eol).strip
      end

      def unindent(startcol, text)
        text.gsub(/^ {0,#{startcol}}/, "")
      end
 
      def store_keyword_content(event, data, p, eof)
        end_point = (!@next_keyword_start or (p == eof)) ? p : @next_keyword_start
        con = yield data[@content_start...end_point].utf8_pack("c*")
        @listener.send(event, @keyword, con, @current_line)
      end
      
      def current_line_content(data, p)
        rest = data[@last_newline..-1]
        rest[0..rest.index(10)||-1].utf8_pack("c*").strip
      end
    end
  end
end<commit_msg>Use constants for regexen and linefeeds<commit_after>require 'gherkin/core_ext/array'

module Gherkin
  module RbLexer
    class <%= @i18n.sanitized_key.capitalize %> #:nodoc:
      %%{
        machine lexer;
 
        action begin_content {
          @content_start = p
          @current_line = @line_number
        }

        action start_pystring {
          @current_line = @line_number
          @start_col = p - @last_newline
        }
        
        action begin_pystring_content {
          @content_start = p
        }

        action store_pystring_content {
          con = unindent(@start_col, data[@content_start...@next_keyword_start-1].utf8_pack("c*").sub(/(\r?\n)?( )*\Z/, ''))
          @listener.py_string(con, @current_line) 
        }
 
        action store_feature_content {
          store_keyword_content(:feature, data, p, eof) { |con| multiline_strip(con) }
          p = @next_keyword_start - 1 if @next_keyword_start          
          @next_keyword_start = nil
        }
 
        action store_background_content {
          store_keyword_content(:background, data, p, eof) { |con| multiline_strip(con) }
          p = @next_keyword_start - 1 if @next_keyword_start                    
          @next_keyword_start = nil
        }

        action store_scenario_content {
          store_keyword_content(:scenario, data, p, eof) { |con| multiline_strip(con) }
          p = @next_keyword_start - 1 if @next_keyword_start                    
          @next_keyword_start = nil
        }

        action store_scenario_outline_content {
          store_keyword_content(:scenario_outline, data, p, eof) { |con| multiline_strip(con) }
          p = @next_keyword_start - 1 if @next_keyword_start                    
          @next_keyword_start = nil
        }

        action store_examples_content {
          store_keyword_content(:examples, data, p, eof) { |con| multiline_strip(con) }
          p = @next_keyword_start - 1 if @next_keyword_start                    
          @next_keyword_start = nil
        }

        action store_step_content {
          con = data[@content_start...p].utf8_pack("c*").strip
          @listener.step(@keyword, con, @current_line)
        }
 
        action store_comment_content {
          con = data[@content_start...p].utf8_pack("c*").strip
          @listener.comment(con, @line_number)
          @keyword_start = nil
        }
 
        action store_tag_content {
          con = data[@content_start...p].utf8_pack("c*").strip
          @listener.tag(con, @current_line)
          @keyword_start = nil
        }
 
        action inc_line_number {
          @line_number += 1
        }
 
        action last_newline {
          @last_newline = p + 1
        }

        action start_keyword {
          @keyword_start ||= p
        }
 
        action end_keyword {
          @keyword = data[@keyword_start...p].utf8_pack("c*").sub(/:$/,'')
          @keyword_start = nil
        }
 
        action next_keyword_start {
          @next_keyword_start = p
        }

        action start_table {
          p = p - 1
          @rows = []
          @current_line = @line_number
        }

        action start_row {
          current_row = []
        }
 
        action begin_cell_content {
          @content_start = p
        }

        action store_cell_content {
          con = data[@content_start...p].utf8_pack("c*").strip
          current_row << con
        }

        action store_row {
          @rows << current_row
        }

        action store_table {
          if @rows.size != 0
            @listener.table(@rows, @current_line)
          end 
        }
 
        action end_feature {
          if cs < lexer_first_final
            content = current_line_content(data, p)
            raise Lexer::LexingError.new("Lexing error on line %d: '%s'." % [@line_number, content])
          end
        }

        include lexer_common "lexer_common.<%= @i18n.sanitized_key %>.rl"; 
      }%%
 
      def initialize(listener)
        @listener = listener
        %% write data;
      end
 
      def scan(data)
        data = (data + "\n%_FEATURE_END_%").unpack("c*") # Explicit EOF simplifies things considerably
        eof = pe = data.length
 
        @line_number = 1
        @last_newline = 0
 
        %% write init;
        %% write exec;
      end

      CRLF_RE = /\r\n/
      LF_RE = /[^\r]\n/
      CRLF = "\r\n"
      LF = "\n"

      def multiline_strip(text)
        crlf_count = text.scan(CRLF_RE).size
        lf_count = text.scan(LF_RE).size
        eol = crlf_count > lf_count ? CRLF : LF
        text.split(/\r?\n/).map{|s| s.strip}.join(eol).strip
      end

      def unindent(startcol, text)
        text.gsub(/^ {0,#{startcol}}/, "")
      end
 
      def store_keyword_content(event, data, p, eof)
        end_point = (!@next_keyword_start or (p == eof)) ? p : @next_keyword_start
        con = yield data[@content_start...end_point].utf8_pack("c*")
        @listener.send(event, @keyword, con, @current_line)
      end
      
      def current_line_content(data, p)
        rest = data[@last_newline..-1]
        rest[0..rest.index(10)||-1].utf8_pack("c*").strip
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.11253644314868805, 'fraction_numerical': 0.0058309037900874635, 'mean_word_length': 1.757502679528403, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5792773', 'n_tokens_mistral': 3337, 'n_tokens_neox': 3123, 'n_words': 596}