<commit_before># encoding: utf-8

module RubyAMI
  class Lexer
    CR                = /\r/, #UNUSED?
    LF                = /\n/, #UNUSED?
    CRLF              = /\r\n/, #UNUSED?
    LOOSE_NEWLINE     = /\r?\n/, #UNUSED?
    WHITE             = /[\t ]/, #UNUSED?
    COLON             = /: */, #UNUSED?
    STANZA_BREAK      = /\r\n\r\n/,
    REST_OF_LINE      = /(.*)?\r\n/, #UNUSED?
    PROMPT            = /Asterisk Call Manager\/(\d+\.\d+)\r\n/,
    KEY               = /([[[:alnum:]][[:print:]]])[\r\n:]+/, #UNUSED?
    KEYVALUEPAIR      = /^([[[:alnum:]]-_ ]+): *(.*)\r\n/,
    FOLLOWSDELIMITER  = /\r?\n?--END COMMAND--\r\n\r\n/,
    RESPONSE          = /response: */i, #UNUSED?
    SUCCESS           = /response: *success\r\n/i,
    PONG              = /response: *pong\r\n/i,
    EVENT             = /event: *(.*)?\r\n/i,
    ERROR             = /response: *error\r\n/i,
    FOLLOWS           = /response: *follows\r\n/i,
    FOLLOWSBODY       = /(.*)?\r?\n?(?:--END COMMAND--\r\n\r\n|\r\n\r\n\r\n)/m
    SCANNER           = /.*?#{STANZA_BREAK}/m
    HEADER_SLICE      = /.*\r\n/

    attr_accessor :ami_version

    def initialize(delegate = nil)
      @delegate = delegate
      @data = ""
      @ami_version = 0.0
    end

    def <<(new_data)
      extend_buffer_with new_data
      parse_buffer
    end

    def parse_buffer
      # Special case for the protocol header
      if @data =~ PROMPT
        @ami_version = $1
        @data.slice!(HEADER_SLICE)
      end

      # We need at least one complete message before parsing
      return unless @data =~ STANZA_BREAK

      @processed = ''

      response_follows_message = false
      current_message = nil
      @data.scan(SCANNER).each do |raw|
        if response_follows_message
          if handle_response_follows response_follows_message, raw
            @processed << raw
            message_received response_follows_message
            response_follows_message = nil
          end
        else
          response_follows_message = parse_message raw
        end<commit_msg>[CS] Don't use a regex where a simple String#include? will do<commit_after># encoding: utf-8

module RubyAMI
  class Lexer
    CR                = /\r/, #UNUSED?
    LF                = /\n/, #UNUSED?
    CRLF              = /\r\n/, #UNUSED?
    LOOSE_NEWLINE     = /\r?\n/, #UNUSED?
    WHITE             = /[\t ]/, #UNUSED?
    COLON             = /: */, #UNUSED?
    STANZA_BREAK      = "\r\n\r\n"
    REST_OF_LINE      = /(.*)?\r\n/, #UNUSED?
    PROMPT            = /Asterisk Call Manager\/(\d+\.\d+)\r\n/,
    KEY               = /([[[:alnum:]][[:print:]]])[\r\n:]+/, #UNUSED?
    KEYVALUEPAIR      = /^([[[:alnum:]]-_ ]+): *(.*)\r\n/,
    FOLLOWSDELIMITER  = /\r?\n?--END COMMAND--\r\n\r\n/,
    RESPONSE          = /response: */i, #UNUSED?
    SUCCESS           = /response: *success\r\n/i,
    PONG              = /response: *pong\r\n/i,
    EVENT             = /event: *(.*)?\r\n/i,
    ERROR             = /response: *error\r\n/i,
    FOLLOWS           = /response: *follows\r\n/i,
    FOLLOWSBODY       = /(.*)?\r?\n?(?:--END COMMAND--\r\n\r\n|\r\n\r\n\r\n)/m
    SCANNER           = /.*?#{STANZA_BREAK}/m
    HEADER_SLICE      = /.*\r\n/

    attr_accessor :ami_version

    def initialize(delegate = nil)
      @delegate = delegate
      @data = ""
      @ami_version = 0.0
    end

    def <<(new_data)
      extend_buffer_with new_data
      parse_buffer
    end

    def parse_buffer
      # Special case for the protocol header
      if @data =~ PROMPT
        @ami_version = $1
        @data.slice!(HEADER_SLICE)
      end

      # We need at least one complete message before parsing
      return unless @data.include?(STANZA_BREAK)

      @processed = ''

      response_follows_message = false
      current_message = nil
      @data.scan(SCANNER).each do |raw|
        if response_follows_message
          if handle_response_follows response_follows_message, raw
            @processed << raw
            message_received response_follows_message
            response_follows_message = nil
          end
        else
          response_follows_message = parse_message raw
        end
{'fraction_non_alphanumeric': 0.15144230769230768, 'fraction_numerical': 0.0019230769230769232, 'mean_word_length': 1.5748762376237624, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '620636', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1446, 'n_words': 286}