<commit_before>require 'levels'
require 'mon_stats'
require 'mon_lvl'

class Array
  def sum
    inject(0) {|a,e| a+e}
  end

  def avg
    sum.to_f / size
  end
end

# Store/load datasources from a marshaled cache for a huge speedup on subsequent runs.
def cache(key, &block)
  cache_file = "cache.#{key}.marshal"
  if File.exist?(cache_file)
    return Marshal.load(File.read(cache_file))
  else
    obj = block.call
    File.open(cache_file,'w') {|f| f.puts Marshal.dump(obj)}
    return obj
  end
end

levels = cache('levels') {Levels.new}
mon_stats = cache('mon_stats') {MonStats.new}
mon_lvl = cache('mon_lvl') {MonLvl.new}

levels.levels.each do |level|
  normal_monster_xps = level[:monsters].map do |mon_id|
    monster = mon_stats[mon_id]
    raise "Monster not found: #{mon_id}" if monster.nil?
    monster_level = monster[:levels][0]
    experience_at_level_in_difficulty = mon_lvl[monster_level][:exp].first
    experience_percentage = monster[:exp_pct].first.to_f/100
    experience_at_level_in_difficulty * experience_percentage
  end

  nightmare_area_level = level[:levels][1]
  nightmare_monster_xps = level[:monsters].map do |mon_id|
    monster = mon_stats[mon_id]
    raise "Monster not found: #{mon_id}" if monster.nil?
    monster_level = nightmare_area_level
    experience_at_level_in_difficulty = mon_lvl[monster_level][:exp].first
    experience_percentage = monster[:exp_pct].first.to_f/100
    experience_at_level_in_difficulty * experience_percentage
  end

  hell_area_level = level[:levels][2]
  hell_monster_xps = level[:monsters].map do |mon_id|
    monster = mon_stats[mon_id]
    raise "Monster not found: #{mon_id}" if monster.nil?
    monster_level = hell_area_level
    experience_at_level_in_difficulty = mon_lvl[monster_level][:exp].first
    experience_percentage = monster[:exp_pct].first.to_f/100
    experience_at_level_in_difficulty * experience_percentage
  end

  level[:exp] = [
    normal_monster_xps.avg.floor,
    nightmare_monster_xps.avg.floor,
    hell_monster_xps.avg.floor
  ]
end

puts levels<commit_msg>Bugfix: use nightmare/hell modifiers to calculate XP for those difficulties<commit_after>require 'levels'
require 'mon_stats'
require 'mon_lvl'

class Array
  def sum
    inject(0) {|a,e| a+e}
  end

  def avg
    sum.to_f / size
  end
end

# Store/load datasources from a marshaled cache for a huge speedup on subsequent runs.
def cache(key, &block)
  cache_file = "cache.#{key}.marshal"
  if File.exist?(cache_file)
    return Marshal.load(File.read(cache_file))
  else
    obj = block.call
    File.open(cache_file,'w') {|f| f.puts Marshal.dump(obj)}
    return obj
  end
end

levels = cache('levels') {Levels.new}
mon_stats = cache('mon_stats') {MonStats.new}
mon_lvl = cache('mon_lvl') {MonLvl.new}

levels.levels.each do |level|
  normal_monster_xps = level[:monsters].map do |mon_id|
    monster = mon_stats[mon_id]
    raise "Monster not found: #{mon_id}" if monster.nil?
    monster_level = monster[:levels][0]
    experience_at_level_in_difficulty = mon_lvl[monster_level][:exp][0]
    experience_percentage = monster[:exp_pct][0].to_f/100
    experience_at_level_in_difficulty * experience_percentage
  end

  nightmare_area_level = level[:levels][1]
  nightmare_monster_xps = level[:monsters].map do |mon_id|
    monster = mon_stats[mon_id]
    raise "Monster not found: #{mon_id}" if monster.nil?
    monster_level = nightmare_area_level
    experience_at_level_in_difficulty = mon_lvl[monster_level][:exp][1]
    experience_percentage = monster[:exp_pct][1].to_f/100
    experience_at_level_in_difficulty * experience_percentage
  end

  hell_area_level = level[:levels][2]
  hell_monster_xps = level[:monsters].map do |mon_id|
    monster = mon_stats[mon_id]
    raise "Monster not found: #{mon_id}" if monster.nil?
    monster_level = hell_area_level
    experience_at_level_in_difficulty = mon_lvl[monster_level][:exp][2]
    experience_percentage = monster[:exp_pct][2].to_f/100
    experience_at_level_in_difficulty * experience_percentage
  end

  level[:exp] = [
    normal_monster_xps.avg.floor,
    nightmare_monster_xps.avg.floor,
    hell_monster_xps.avg.floor
  ]
end

puts levels
{'fraction_non_alphanumeric': 0.10796545105566219, 'fraction_numerical': 0.007677543186180422, 'mean_word_length': 4.656716417910448, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '270818', 'n_tokens_mistral': 1546, 'n_tokens_neox': 1509, 'n_words': 292}