<commit_before>module OpenNlp
  class Parser < Tool
    # Initializes new instance of Parser
    #
    # @param [OpenNlp::Model::Parser] parser_model
    # @param [OpenNlp::Model::Tokenizer] token_model
    def initialize(parser_model, token_model)
      unless parser_model.is_a?(OpenNlp::Model)
        fail ArgumentError, 'parser_model must be an OpenNlp::Model'
      end

      unless token_model.is_a?(Model::Tokenizer)
        fail ArgumentError, 'token_model must be an OpenNlp::Tokenizer::Model'
      end

      @j_instance = Java::opennlp.tools.parser.ParserFactory.create(parser_model.j_model)
      @tokenizer = Tokenizer.new(token_model)
    end

    # Parses text into instance of Parse class
    #
    # @param [String] text text to parse
    # @return [OpenNlp::Parser::Parse]
    def parse(text)
      raise ArgumentError, 'passed text must be a String' unless text.is_a?(String)
      text.empty? ? {} : parse_tokens(tokenizer.tokenize(text), text)
    end

    private

    attr_reader :tokenizer

    def get_token_offset(text, tokens, index)
      offset = 0
      return offset unless index > 0

      for i in (1..index) do
        offset = text.index tokens[i], offset + tokens[i - 1].size
      end
      offset
    end

    def build_parse_obj(text, span_start, span_end, type=Java::opennlp.tools.parser.AbstractBottomUpParser::INC_NODE, probability=1, token_index=0)
      Java::opennlp.tools.parser.Parse.new(
        text.to_java(:String),
        Java::opennlp.tools.util.Span.new(span_start, span_end),
        type.to_java(:String),
        probability.to_java(:Double), # probability ?
        token_index.to_java(:Integer) # the token index of the head of this parse
      )
    end

    def parse_tokens(tokens, text)
      parse_obj = build_parse_obj(text, 0, text.size)
      parse_type = Java::opennlp.tools.parser.AbstractBottomUpParser::TOK_NODE

      tokens.each_with_index do |tok, i|
        start = get_token_offset(text, tokens, i)
        token_parse = build_parse_obj(text, start, start + tok.size, parse_type, 0, i)
        parse_obj.insert(token_parse)
      end

      Parser::Parse.new(j_instance.parse(parse_obj))
    end
  end
end
<commit_msg>Check that parser_model in an instance of Model::Parser in Parser#initialize.<commit_after>module OpenNlp
  class Parser < Tool
    # Initializes new instance of Parser
    #
    # @param [OpenNlp::Model::Parser] parser_model
    # @param [OpenNlp::Model::Tokenizer] token_model
    def initialize(parser_model, token_model)
      unless parser_model.is_a?(OpenNlp::Model::Parser)
        fail ArgumentError, 'parser_model must be an OpenNlp::Model'
      end

      unless token_model.is_a?(Model::Tokenizer)
        fail ArgumentError, 'token_model must be an OpenNlp::Tokenizer::Model'
      end

      @j_instance = Java::opennlp.tools.parser.ParserFactory.create(parser_model.j_model)
      @tokenizer = Tokenizer.new(token_model)
    end

    # Parses text into instance of Parse class
    #
    # @param [String] text text to parse
    # @return [OpenNlp::Parser::Parse]
    def parse(text)
      raise ArgumentError, 'passed text must be a String' unless text.is_a?(String)
      text.empty? ? {} : parse_tokens(tokenizer.tokenize(text), text)
    end

    private

    attr_reader :tokenizer

    def get_token_offset(text, tokens, index)
      offset = 0
      return offset unless index > 0

      for i in (1..index) do
        offset = text.index tokens[i], offset + tokens[i - 1].size
      end
      offset
    end

    def build_parse_obj(text, span_start, span_end, type=Java::opennlp.tools.parser.AbstractBottomUpParser::INC_NODE, probability=1, token_index=0)
      Java::opennlp.tools.parser.Parse.new(
        text.to_java(:String),
        Java::opennlp.tools.util.Span.new(span_start, span_end),
        type.to_java(:String),
        probability.to_java(:Double), # probability ?
        token_index.to_java(:Integer) # the token index of the head of this parse
      )
    end

    def parse_tokens(tokens, text)
      parse_obj = build_parse_obj(text, 0, text.size)
      parse_type = Java::opennlp.tools.parser.AbstractBottomUpParser::TOK_NODE

      tokens.each_with_index do |tok, i|
        start = get_token_offset(text, tokens, i)
        token_parse = build_parse_obj(text, start, start + tok.size, parse_type, 0, i)
        parse_obj.insert(token_parse)
      end

      Parser::Parse.new(j_instance.parse(parse_obj))
    end
  end
end

{'fraction_non_alphanumeric': 0.10205908683974933, 'fraction_numerical': 0.0035810205908683975, 'mean_word_length': 3.172735760971055, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '79185', 'n_tokens_mistral': 1443, 'n_tokens_neox': 1412, 'n_words': 327}