<commit_before>module PointsExtraction
  COPULAE = %w(act appear be become come end get go grow fall feel keep look prove remain run seem smell sound stay taste turn wax)

  def self.points_for_matches(neo4j_client, matches, keys)
    points = matches.map do |verb, results|
      results.reject! { |r| r.first.to_a.compact.size < 2 }
      next if results.empty?
      string = neo4j_client.permitted_descendant_string(verb, PointsExtraction::COPULAE.include?(verb.lemma))
      results.map do |r|
        result = {}
        match = Hash[r.first.to_h.to_a.reject { |e| e.last.nil? }]
        result.merge!(string: string) if keys.include? 'string'
        result.merge!(match: match) if keys.include? 'match'
        result.merge!(pattern: match.map { |k, v| "#{v.lemma}.#{k}" }.join(" ")) if keys.include? 'pattern'
        result.merge!(frames: r.last.map { |f| f[:frames] }.uniq) if keys.include? 'frames'
        result
      end
    end.flatten.compact
  end

  def self.matches_for_verbs(neo4j_client, frames, frame_queries)
    verb_queries = neo4j_client.verbs.map do |v|
      queries_for_verb(v, frames, frame_queries).map do |q|
        { uuid: v.uuid, query: q.first, frames: q.last }
      end
    end.flatten.group_by { |e| e[:query] }

    verb_queries.map do |shared_query, frame_queries|
      shared_query = shared_query.gsub("UUIDS", frame_queries.map { |q| q[:uuid] }.join('", "'))
      neo4j_client.execute(shared_query).to_a.map { |r| [r, frame_queries] }
    end.flatten(1).compact.group_by { |r| r.first.verb }.reject { |_, v| v.empty? }
  end

  private<commit_msg>Only return short points if they are from a frame<commit_after>module PointsExtraction
  COPULAE = %w(act appear be become come end get go grow fall feel keep look prove remain run seem smell sound stay taste turn wax)

  def self.points_for_matches(neo4j_client, matches, keys)
    points = matches.map do |verb, results|
      results.reject! { |r| r.first.to_a.compact.size < 2 }
      next if results.empty?
      string = neo4j_client.permitted_descendant_string(verb, PointsExtraction::COPULAE.include?(verb.lemma))
      results.map do |r|
        next if r.last.map { |f| f[:frames] }.uniq == [:generic] && r.first.to_a.compact.size < 3
        result = {}
        match = Hash[r.first.to_h.to_a.reject { |e| e.last.nil? }]
        result.merge!(string: string) if keys.include? 'string'
        result.merge!(match: match) if keys.include? 'match'
        result.merge!(pattern: match.map { |k, v| "#{v.lemma}.#{k}" }.join(" ")) if keys.include? 'pattern'
        result.merge!(frames: r.last.map { |f| f[:frames] }.uniq) if keys.include? 'frames'
        result
      end
    end.flatten.compact
  end

  def self.matches_for_verbs(neo4j_client, frames, frame_queries)
    verb_queries = neo4j_client.verbs.map do |v|
      queries_for_verb(v, frames, frame_queries).map do |q|
        { uuid: v.uuid, query: q.first, frames: q.last }
      end
    end.flatten.group_by { |e| e[:query] }

    verb_queries.map do |shared_query, frame_queries|
      shared_query = shared_query.gsub("UUIDS", frame_queries.map { |q| q[:uuid] }.join('", "'))
      neo4j_client.execute(shared_query).to_a.map { |r| [r, frame_queries] }
    end.flatten(1).compact.group_by { |r| r.first.verb }.reject { |_, v| v.empty? }
  end

  private
{'fraction_non_alphanumeric': 0.14216867469879518, 'fraction_numerical': 0.004518072289156626, 'mean_word_length': 3.2468030690537084, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2746144', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1173, 'n_words': 275}