<commit_before>%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  unsigned_integer = digit+;

  identifier_start = alpha | 0x80..0xff;
  identifier_part  = identifier_start | digit | '""';
  identifier_body  = identifier_start ('_'? identifier_part)*;
  identifier       = '"' identifier_body '"';

  main := |*
    space;

    # Keywords
    'SELECT'         { emit('select'); };
    'FROM'           { emit('from');   };
    'AS'             { emit('as');     };

    # Literals
    'TRUE'           { emit('true');  };
    'FALSE'          { emit('false'); };
    'NULL'           { emit('null');  };

    # Operators
    '('              { emit('left_paren');  };
    ')'              { emit('right_paren'); };
    ','              { emit('comma');       };
    '.'              { emit('period');      };
    '+'              { emit('plus_sign');   };
    '-'              { emit('minus_sign');  };
    '*'              { emit('asterisk');    };
    '/'              { emit('solidus');     };
    'E'              { emit('E');           };

    unsigned_integer { emit_unsigned_integer(); };
    identifier       { emit_identifier();       };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

#    CHUNK_SIZE = 1024
    CHUNK_SIZE = 1  # XXX: 1 used for testing

    %% write data;

    def initialize(input)
      @input = input
      @block = nil
      @buffer = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block

      # Tokenize a stream of input
      while chunk = read_chunk
        @buffer.concat(chunk)
        %% write exec;
        optimize_buffer
      end
      assert_valid_input
      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def emit(token, value = text)
      @block.call(token.to_sym, value)
      @ts = nil
    end

    def emit_unsigned_integer
      emit(:unsigned_integer, Integer(text))
    end

    def emit_identifier
      identifier = text
      identifier.gsub!(/\A"(.*)"\z/, '\1') and identifier.gsub!('""', '"')
      emit(:identifier, identifier)
    end

    def text
      @buffer[@ts...@te].pack('U*')
    end

    def read_chunk
      chunk  = @input.read_nonblock(CHUNK_SIZE)
      format = chunk.encoding == Encoding::UTF_8 ? 'U*' : 'C*'
      chunk.unpack(format)
    rescue IO::WaitReadable
      IO.select([@input])
      retry<commit_msg>Move input scanning into a separate method<commit_after>%%{
  # TODO: extract machine into separate rl file (minus ruby variable config)
  machine sql;

  unsigned_integer = digit+;

  identifier_start = alpha | 0x80..0xff;
  identifier_part  = identifier_start | digit | '""';
  identifier_body  = identifier_start ('_'? identifier_part)*;
  identifier       = '"' identifier_body '"';

  main := |*
    space;

    # Keywords
    'SELECT'         { emit('select'); };
    'FROM'           { emit('from');   };
    'AS'             { emit('as');     };

    # Literals
    'TRUE'           { emit('true');  };
    'FALSE'          { emit('false'); };
    'NULL'           { emit('null');  };

    # Operators
    '('              { emit('left_paren');  };
    ')'              { emit('right_paren'); };
    ','              { emit('comma');       };
    '.'              { emit('period');      };
    '+'              { emit('plus_sign');   };
    '-'              { emit('minus_sign');  };
    '*'              { emit('asterisk');    };
    '/'              { emit('solidus');     };
    'E'              { emit('E');           };

    unsigned_integer { emit_unsigned_integer(); };
    identifier       { emit_identifier();       };
  *|;

  # TODO: keep ruby specific variables inline while moving previous code
  access @;
  variable data @buffer;
  variable p    @p;
  variable pe   self.pe;
  variable eof  @eof;
}%%

module SQL
  class Scanner
    class InvalidInputError < StandardError

      def initialize(input)
        input.rewind
        super("Error tokenizing: #{input.read}")
      end

    end # InvalidInputError

    include Enumerable

#    CHUNK_SIZE = 1024
    CHUNK_SIZE = 1  # XXX: 1 used for testing

    %% write data;

    def initialize(input)
      @input = input
      @block = nil
      @buffer = []
      %% write init;
    end

    def each(&block)
      return to_enum unless block
      return self if @block  # only allow one pass
      @block = block
      scan_input
      self
    end

  protected

    def pe
      @buffer.length
    end

  private

    def scan_input
      while chunk = read_chunk
        @buffer.concat(chunk)
        %% write exec;
        optimize_buffer
      end
      assert_valid_input
    end

    def emit(token, value = text)
      @block.call(token.to_sym, value)
      @ts = nil
    end

    def emit_unsigned_integer
      emit(:unsigned_integer, Integer(text))
    end

    def emit_identifier
      identifier = text
      identifier.gsub!(/\A"(.*)"\z/, '\1') and identifier.gsub!('""', '"')
      emit(:identifier, identifier)
    end

    def text
      @buffer[@ts...@te].pack('U*')
    end

    def read_chunk
      chunk  = @input.read_nonblock(CHUNK_SIZE)
      format = chunk.encoding == Encoding::UTF_8 ? 'U*' : 'C*'
      chunk.unpack(format)
    rescue IO::WaitReadable
      IO.select([@input])
      retry
{'fraction_non_alphanumeric': 0.1351304347826087, 'fraction_numerical': 0.0041739130434782605, 'mean_word_length': 1.8654708520179373, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2186515', 'n_tokens_mistral': 1789, 'n_tokens_neox': 1723, 'n_words': 406}