<commit_before>module ParallelSpecs
  extend self

  # finds all specs and partitions them into groups
  def specs_in_groups(root, num)
    specs = (Dir["#{root}/spec/**/*_spec.rb"]).sort
    return [ specs ] if num == 1
              
    specs_with_sizes, total_size = find_sizes(specs)

    groups = []
    num.times { |i| groups[i] = [] }
    
    group_size = (total_size / num.to_f)
        
    i = 0
    current_size = 0
    specs_with_sizes.each do |spec|
      current_size += spec[0]
      i += 1 if current_size > group_size * (i+1)     
      groups[i] << spec[1]
    end
    
    groups
  end

  def run_tests(test_files, process_number)
    cmd = "export RAILS_ENV=test ; export TEST_ENV_NUMBER=#{process_number==0?'':process_number+1} ; export RSPEC_COLOR=1 ; script/spec -O spec/spec.opts #{test_files*' '}"
    f = open("|#{cmd}")
    while out = f.gets(".")
      print out
    end
  end
  
  private
  
  def find_sizes(specs)
    total_size = 0
    specs_with_sizes = []
    specs.each do |file|
      size = File.stat(file).size
      specs_with_sizes << [ size, file ]
      total_size += size
    end
    return specs_with_sizes, total_size    
  end
end<commit_msg>Removed test output delay caused by not flushing STDOUT.<commit_after>module ParallelSpecs
  extend self

  # finds all specs and partitions them into groups
  def specs_in_groups(root, num)
    specs = (Dir["#{root}/spec/**/*_spec.rb"]).sort
    return [ specs ] if num == 1
              
    specs_with_sizes, total_size = find_sizes(specs)

    groups = []
    num.times { |i| groups[i] = [] }
    
    group_size = (total_size / num.to_f)
        
    i = 0
    current_size = 0
    specs_with_sizes.each do |spec|
      current_size += spec[0]
      i += 1 if current_size > group_size * (i+1)     
      groups[i] << spec[1]
    end
    
    groups
  end

  def run_tests(test_files, process_number)
    cmd = "export RAILS_ENV=test ; export TEST_ENV_NUMBER=#{process_number==0?'':process_number+1} ; export RSPEC_COLOR=1 ; script/spec -O spec/spec.opts #{test_files*' '}"
    f = open("|#{cmd}")
    while out = f.gets(".")
      print out
      STDOUT.flush
    end
  end
  
  private
  
  def find_sizes(specs)
    total_size = 0
    specs_with_sizes = []
    specs.each do |file|
      size = File.stat(file).size
      specs_with_sizes << [ size, file ]
      total_size += size
    end
    return specs_with_sizes, total_size    
  end
end
{'fraction_non_alphanumeric': 0.11793611793611794, 'fraction_numerical': 0.009009009009009009, 'mean_word_length': 2.4800569800569803, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6398708', 'n_tokens_mistral': 898, 'n_tokens_neox': 874, 'n_words': 202}