<commit_before># A ruby script to display the count of dojos by size
# So the output below means there were 45 dojos with 0 traffic lights
# and 55 dojos with 1 traffic light, etc.
#
# ruby stats.rb
# 0 45
# 1 55
# 2 42
# ...
# 156 1
# 194 1

def avatars
  %w(
      alligator buffalo cheetah deer
      elephant frog gorilla hippo
      koala lion moose panda
      raccoon snake wolf zebra
    )
end

def traffic_light_count(kata_dir)
  count = 0
  avatars.each do |avatar|
    inc_filename = "#{kata_dir}/#{avatar}/increments.rb"
    if File.exists? inc_filename
      incs = eval IO.popen("cat #{inc_filename}").read
      count += incs.length
    end
  end
  count
end

stats = { }
index = eval IO.popen('cat katas/index.rb').read
index.each do |entry|
  begin
    id = entry[:id]
    created = Time.mktime(*entry[:created])
    days_old = ((Time.now - created) / 60 / 60 / 24).to_i

    inner_dir = id[0..1]   
    outer_dir = id[2..9]
    kata_dir = "katas/#{inner_dir}/#{outer_dir}"
    manifest_name = "#{kata_dir}/manifest.rb"
    if File.exists?(manifest_name)
      count = traffic_light_count(kata_dir)
      stats[count] ||= 0 
      stats[count] += 1
    end
  end    
end

stats.sort.each do |count,tally|
  printf("%3d %3d\n", count, tally)
end<commit_msg>stats.rb script now shows number of avatars in dojos<commit_after># A ruby script to display the count of dojos by size
# So the output below means there were 45 dojos with 0 traffic lights
# and 55 dojos with 1 traffic light, etc.
# The numbers following inside [ ] are the number of avatars in the dojos
# So for example
#    34 2 [5,6] means there were 2 dojos with 34 traffic lights
#               and those two dojos had 5 and 6 avatars respetively.
#
# ruby stats.rb
# 0 45
# 1 55 [...]
# 2 42 [...]
# ...
# 34 2 [5, 6]
# 156 1 [8]
# 194 1 [3]

def avatars
  %w(
      alligator buffalo cheetah deer
      elephant frog gorilla hippo
      koala lion moose panda
      raccoon snake wolf zebra
    )
end

def traffic_light_count(kata_dir)
  tally,count = 0,0
  avatars.each do |avatar|
    inc_filename = "#{kata_dir}/#{avatar}/increments.rb"
    if File.exists? inc_filename
      incs = eval IO.popen("cat #{inc_filename}").read
      if incs.length > 0
        tally += 1
        count += incs.length
      end
    end
  end
  [tally,count]
end

stats = { }
index = eval IO.popen('cat katas/index.rb').read
index.each do |entry|
  begin
    id = entry[:id]
    created = Time.mktime(*entry[:created])
    days_old = ((Time.now - created) / 60 / 60 / 24).to_i

    inner_dir = id[0..1]   
    outer_dir = id[2..9]
    kata_dir = "katas/#{inner_dir}/#{outer_dir}"
    manifest_name = "#{kata_dir}/manifest.rb"
    if File.exists?(manifest_name)
      tally,count = traffic_light_count(kata_dir)
      stats[count] ||= [ ]
      stats[count] << tally
    end
  end    
end

stats.sort.each do |count,tallies|
  printf("%3d %3d",count, tallies.length)
  sep = ""
  if tallies.length <= 20
    printf(" [")
    tallies.sort.each do |n|
      printf("%s%d", sep, n)
      sep = ", "
    end
    printf("]\n")
  else
    printf("\n")
  end
end
{'fraction_non_alphanumeric': 0.12343699903815325, 'fraction_numerical': 0.030779095864058994, 'mean_word_length': 2.6236933797909407, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2571439', 'n_tokens_mistral': 1220, 'n_tokens_neox': 1124, 'n_words': 337}