<commit_before>require 'net/http'

module Curator
  class CoreNLPClient
    def initialize(host)
      params = URI.encode('properties={"annotators": "lemma,tokenize,ssplit,depparse"}')

      @uri = URI(host + "/?" + params)
      @http_client = Net::HTTP.new(@uri.host, @uri.port)
    end

    def request_parse(text)
      req = Net::HTTP::Post.new(@uri)
      req.body = text
      JSON.parse(@http_client.request(req).body)['sentences'].map do |sentence|
        [sentence['tokens'], sentence['collapsed-ccprocessed-dependencies']]
      end
    end
  end

  def self.select_best(points, return_group=false)
    points = reparse_points(points.uniq { |p| p["String"] })
    points = permitted(points)
    # return_group ? return points : return points.sample
    if points.empty?
      return_group ? (return []) : (return nil)
    end

    group_bigrams = []
    points.each do |p|
      group_bigrams += bigrams(p["String"])
    end
    group_bigrams = Utils.sorted_dup_hash(group_bigrams)
    points = points.sort_by do |p|
      clean = p["String"].downcase.gsub(/[^\s\w']/, "").gsub(/\s+/, " ").strip
      score = 0
      group_bigrams.each do |k, v|
        score += v if clean.include? k
      end
      p["Score"] = score.to_f / clean.split(" ").size
    end
    return points if return_group
    #return nil unless points.last
    #if original_points.size.between?(4, 8) && points.last["Components"].size > 2 && points.map { |p| p["Components"] }.uniq.size == 1
      #p original_points.first["Components"]
      #puts original_points.map {|p| Presenter.clean(p["String"]) }.uniq
      #print " > "
      #puts Presenter.clean(points.last["String"])
      #puts "------"
      #exit
    #end
    return points.last
  end

  def self.select_best_question(points)
    points.reject! do |p|
      p["String"].strip[-1] != "?"
    end
    points.sort_by {|x|x["String"].length}.first
  end

  def self.bigrams(string)
    words = string.downcase.gsub(/[^\s\w']/, "").split(/\s+/)
    words.each_with_index.to_a[0..-2].map do |e, i|
      [e, words[i+1]].join(" ")
    end
  end

  def self.reparse_points(points)
    cnc = CoreNLPClient.new("http://local.docker:9000")
    points.map do |point|
      parse = cnc.request_parse(clean_string(point["String"])).first
      point["Relations"], point["Lemmas"] = relations_and_lemmas_from_parse(parse)
      point
    end
  end

  def self.permitted(points)
    points.reject do |point|
      !point["String"].length.between?(15, 100) ||
      #point["String"].match(/^(if|and|but|or|just|after|before|their|his|her|why)/i) ||
      point["String"].match(/this|And/) ||
      point["String"].match(/^\w+ ?, /) ||
      point["String"].match(/(\s*,){2}/) ||
      point["String"].match(/([A-Z]+ ){2,}/) ||
      point["String"].scan(/[a-z]+[0-9A-Z]/).any? ||
      point["String"].scan(/[LR]RB|[LR]SB/).size.odd? ||
      point["Relations"].count { |r| r.match(/dep|acl|csubj|ccomp/) } > 0 ||
      point["Relations"].count { |r| r.match(/advcl/) } > 1 ||
      point["Relations"].count { |r| r.match(/conj|nmod/) } > 2 ||
      point["Relations"].join(" ").match(/amod (punct)?$/) ||
      contains_case_change(point) ||
      contains_bad_it(point) ||
      bad_repeated_word(point) ||
      boring_words(point, 2)
    end
  end

  def self.clean_string(string, question=false)
    string = string.strip
      .gsub(/^[^\w\{]+/, "")
      .gsub(/ ?- ?\.?$/, "")
      .gsub(/[\s;\.,]+$/, "")
      .gsub(/\s+(n?[,'\.])/, '\1')
      .gsub("-LRB-", "(").gsub("-RRB-", ")")
      .gsub("( ", "(").gsub(" )", ")")
      .gsub("-LSB-", "[").gsub("-RSB-", "]")
      .gsub("[ ", "[").gsub(" ]", "]").strip<commit_msg>Reject extracts that end in 'is'<commit_after>require 'net/http'

module Curator
  class CoreNLPClient
    def initialize(host)
      params = URI.encode('properties={"annotators": "lemma,tokenize,ssplit,depparse"}')

      @uri = URI(host + "/?" + params)
      @http_client = Net::HTTP.new(@uri.host, @uri.port)
    end

    def request_parse(text)
      req = Net::HTTP::Post.new(@uri)
      req.body = text
      JSON.parse(@http_client.request(req).body)['sentences'].map do |sentence|
        [sentence['tokens'], sentence['collapsed-ccprocessed-dependencies']]
      end
    end
  end

  def self.select_best(points, return_group=false)
    points = reparse_points(points.uniq { |p| p["String"] })
    points = permitted(points)
    # return_group ? return points : return points.sample
    if points.empty?
      return_group ? (return []) : (return nil)
    end

    group_bigrams = []
    points.each do |p|
      group_bigrams += bigrams(p["String"])
    end
    group_bigrams = Utils.sorted_dup_hash(group_bigrams)
    points = points.sort_by do |p|
      clean = p["String"].downcase.gsub(/[^\s\w']/, "").gsub(/\s+/, " ").strip
      score = 0
      group_bigrams.each do |k, v|
        score += v if clean.include? k
      end
      p["Score"] = score.to_f / clean.split(" ").size
    end
    return points if return_group
    #return nil unless points.last
    #if original_points.size.between?(4, 8) && points.last["Components"].size > 2 && points.map { |p| p["Components"] }.uniq.size == 1
      #p original_points.first["Components"]
      #puts original_points.map {|p| Presenter.clean(p["String"]) }.uniq
      #print " > "
      #puts Presenter.clean(points.last["String"])
      #puts "------"
      #exit
    #end
    return points.last
  end

  def self.select_best_question(points)
    points.reject! do |p|
      p["String"].strip[-1] != "?"
    end
    points.sort_by {|x|x["String"].length}.first
  end

  def self.bigrams(string)
    words = string.downcase.gsub(/[^\s\w']/, "").split(/\s+/)
    words.each_with_index.to_a[0..-2].map do |e, i|
      [e, words[i+1]].join(" ")
    end
  end

  def self.reparse_points(points)
    cnc = CoreNLPClient.new("http://local.docker:9000")
    points.map do |point|
      parse = cnc.request_parse(clean_string(point["String"])).first
      point["Relations"], point["Lemmas"] = relations_and_lemmas_from_parse(parse)
      point
    end
  end

  def self.permitted(points)
    points.reject do |point|
      !point["String"].length.between?(15, 100) ||
      #point["String"].match(/^(if|and|but|or|just|after|before|their|his|her|why)/i) ||
      point["String"].match(/is\W+$/) ||
      point["String"].match(/this|And/) ||
      point["String"].match(/^\w+ ?, /) ||
      point["String"].match(/(\s*,){2}/) ||
      point["String"].match(/([A-Z]+ ){2,}/) ||
      point["String"].scan(/[a-z]+[0-9A-Z]/).any? ||
      point["String"].scan(/[LR]RB|[LR]SB/).size.odd? ||
      point["Relations"].count { |r| r.match(/dep|acl|csubj|ccomp/) } > 0 ||
      point["Relations"].count { |r| r.match(/advcl/) } > 1 ||
      point["Relations"].count { |r| r.match(/conj|nmod/) } > 2 ||
      point["Relations"].join(" ").match(/amod (punct)?$/) ||
      contains_case_change(point) ||
      contains_bad_it(point) ||
      bad_repeated_word(point) ||
      boring_words(point, 2)
    end
  end

  def self.clean_string(string, question=false)
    string = string.strip
      .gsub(/^[^\w\{]+/, "")
      .gsub(/ ?- ?\.?$/, "")
      .gsub(/[\s;\.,]+$/, "")
      .gsub(/\s+(n?[,'\.])/, '\1')
      .gsub("-LRB-", "(").gsub("-RRB-", ")")
      .gsub("( ", "(").gsub(" )", ")")
      .gsub("-LSB-", "[").gsub("-RSB-", "]")
      .gsub("[ ", "[").gsub(" ]", "]").strip
{'fraction_non_alphanumeric': 0.21687398593834506, 'fraction_numerical': 0.007301243915630071, 'mean_word_length': 2.8286749482401654, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4473707', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2664, 'n_words': 435}