<commit_before>class Remarkovable
  attr_accessor :markov_model

  def initialize(string: string, prefix_length: 2)
    return if string.nil?
    build_markov_model(string, prefix_length)
  end

  def speak(custom_key: nil)
    return if @markov_model.nil?
    key = assign_key(custom_key)
    output = Array(key.capitalize.split(' '))

    until (output & %w(. ! ?)).any?
      match = @markov_model[key].sample
      if match.nil?
        key = @markov_model.keys.sample
        next
      end
      output << match
      output.flatten!
      key = [output[-2], output[-1]].join(' ')
    end

    output.join(' ').gsub(/\s+([,.!?])/, '\1')
  end

  private

  def build_markov_model(string, prefix_length)
    @markov_model = Hash.new { |hash, key| hash[key] = [] }
    words = string.split(/([.!?])|\s+/)
    words.each_with_index do |word, i|
      key = [word]

      (prefix_length - 1).times do |n|
        next_word = i + n + 1
        key << words[next_word]
      end

      key = key.join(' ')
      match = words[i + prefix_length]
      add_triad(key: key, match: match) if i < words.size - prefix_length
    end
  end

  def assign_key(custom_key)
    if !custom_key.nil? && @markov_model.include?(custom_key)
      custom_key
    else
      @markov_model.keys.sample
    end
  end

  def add_triad(key: key, match: match)
    @markov_model[key] = {} unless @markov_model[key]
    @markov_model[key] += [match]
  end
end
<commit_msg>Fix circular reference warning<commit_after>class Remarkovable
  attr_accessor :markov_model

  def initialize(string:, prefix_length: 2)
    return if string.nil?
    build_markov_model(string, prefix_length)
  end

  def speak(custom_key: nil)
    return if @markov_model.nil?
    key = assign_key(custom_key)
    output = Array(key.capitalize.split(' '))

    until (output & %w(. ! ?)).any?
      match = @markov_model[key].sample
      if match.nil?
        key = @markov_model.keys.sample
        next
      end
      output << match
      output.flatten!
      key = [output[-2], output[-1]].join(' ')
    end

    output.join(' ').gsub(/\s+([,.!?])/, '\1')
  end

  private

  def build_markov_model(string, prefix_length)
    @markov_model = Hash.new { |hash, key| hash[key] = [] }
    words = string.split(/([.!?])|\s+/)
    words.each_with_index do |word, i|
      key = [word]

      (prefix_length - 1).times do |n|
        next_word = i + n + 1
        key << words[next_word]
      end

      key = key.join(' ')
      match = words[i + prefix_length]
      add_triad(key: key, match: match) if i < words.size - prefix_length
    end
  end

  def assign_key(custom_key)
    if !custom_key.nil? && @markov_model.include?(custom_key)
      custom_key
    else
      @markov_model.keys.sample
    end
  end

  def add_triad(key:, match:)
    @markov_model[key] = {} unless @markov_model[key]
    @markov_model[key] += [match]
  end
end

{'fraction_non_alphanumeric': 0.14019337016574585, 'fraction_numerical': 0.004143646408839779, 'mean_word_length': 2.616729088639201, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5244272', 'n_tokens_mistral': 1059, 'n_tokens_neox': 1031, 'n_words': 226}