<commit_before>class Splitter
  attr_accessor :string, :regex
  def initialize(string)
    @string = string
    @regex = /[^\.!?\;]+[\.!?\;]"* |[^\n]+\n/
  end

  # Splits a string up on periods followed by spaces, newlines, or periods 
  # followed by newlines. Uses #scan so the matched characters are preserved.
  # Appends a newline to the end of the string if there isn't one already.
  def split
    to_split = string
    unless to_split[-1] == "\n"
      to_split << "\n"
    end
    to_split.scan regex
  end
end<commit_msg>shuffle lines in text file before output<commit_after>class Splitter
  attr_accessor :string, :regex
  def initialize(string)
    @string = string
    @regex = /[^\.!?\;]+[\.!?\;]"* |[^\n]+\n/
  end

  # Splits a string up on periods followed by spaces, newlines, or periods 
  # followed by newlines. Uses #scan so the matched characters are preserved.
  # Appends a newline to the end of the string if there isn't one already.
  def split
    to_split = string
    unless to_split[-1] == "\n"
      to_split << "\n"
    end
    to_split = to_split.split("\n").shuffle.join("\n")
    to_split.scan regex
  end
end
{'fraction_non_alphanumeric': 0.1273996509598604, 'fraction_numerical': 0.0017452006980802793, 'mean_word_length': 2.941580756013746, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5658153', 'n_tokens_mistral': 393, 'n_tokens_neox': 376, 'n_words': 133}