<commit_before>module WordsCounted
  class Counter
    attr_reader :words, :word_occurrences, :word_lengths, :char_count

    WORD_REGEXP = /[\p{Alpha}\-']+/

    def initialize(string, options = {})
      @options = options
      @char_count = string.length
      exclude = filter_proc(options[:exclude])
      @words = string.scan(regexp).reject { |word| exclude.call(word) }
      @word_occurrences = words.each_with_object(Hash.new(0)) do |word, hash|
        hash[word.downcase] += 1
      end
      @word_lengths = words.each_with_object({}) { |word, hash| hash[word] ||= word.length }
    end

    def word_count
      words.size
    end

    def unique_word_count
      words.uniq.size
    end

    def average_chars_per_word
      (char_count / word_count).round(2)
    end

    def most_occurring_words
      highest_ranking word_occurrences
    end

    def longest_words
      highest_ranking word_lengths
    end

    def word_density
      word_occurrences.each_with_object({}) do |(word, occ), hash|
        hash[word] = percent_of(occ)
      end.sort_by { |_, value| value }.reverse
    end

    private

    def highest_ranking(entries)
      entries.group_by { |word, value| value }.sort.last.last
    end

    def percent_of(n)
      (n.to_f / word_count.to_f * 100.0).round(2)
    end

    def regexp
      @options[:regexp] || WORD_REGEXP
    end

    def filter_proc(filter)
      if filter.respond_to?(:to_a)<commit_msg>syntax change for style guide<commit_after>module WordsCounted
  class Counter
    attr_reader :words, :word_occurrences, :word_lengths, :char_count

    WORD_REGEXP = /[\p{Alpha}\-']+/

    def initialize(string, options = {})
      @options = options
      @char_count = string.length
      exclude = filter_proc(options[:exclude])
      @words = string.scan(regexp).reject { |word| exclude.call(word) }
      @word_occurrences = words.each_with_object(Hash.new(0)) do |word, hash|
        hash[word.downcase] += 1
      end
      @word_lengths = words.each_with_object({}) { |word, hash| hash[word] ||= word.length }
    end

    def word_count
      words.size
    end

    def unique_word_count
      words.uniq.size
    end

    def average_chars_per_word
      (char_count / word_count).round(2)
    end

    def most_occurring_words
      highest_ranking word_occurrences
    end

    def longest_words
      highest_ranking word_lengths
    end

    def word_density
      word_occurrences.each_with_object({}) do |(word, occ), hash|
        hash[word] = percent_of(occ)
      end.sort_by { |_, value| value }.reverse
    end

  private

    def highest_ranking(entries)
      entries.group_by { |word, value| value }.sort.last.last
    end

    def percent_of(n)
      (n.to_f / word_count.to_f * 100.0).round(2)
    end

    def regexp
      @options[:regexp] || WORD_REGEXP
    end

    def filter_proc(filter)
      if filter.respond_to?(:to_a)
{'fraction_non_alphanumeric': 0.10620689655172413, 'fraction_numerical': 0.005517241379310344, 'mean_word_length': 2.714468629961588, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2985419', 'n_tokens_mistral': 1024, 'n_tokens_neox': 955, 'n_words': 203}