<commit_before>module Wdpa::GlobalStatsImporter
  extend self

  GLOBAL_STATS_CSV = Rails.root.join('lib/data/seeds/global_stats.csv').freeze

  def self.import
    GlobalStatistic.destroy_all

    attrs = {singleton_guard: 0}
    CSV.foreach(GLOBAL_STATS_CSV, headers: true) do |row|
      field = row['type']
      value = parse_value(row['value'])
      attrs.merge!("#{field}": value)
    end
    
    GlobalStatistic.first_or_create(attrs)
  end

  private

  # If it's a string, ensure to remove commas before casting to float.
  # If it's a float this will basically return the value as it is in the csv.
  # Even though strings in the csv are mostly integers, converting it to float here
  # shouldn't cause issues with the database where the field is explicitly an integer.
  # Postgres should take care of it.
  def self.parse_value(val)
    val.to_s.split(',').join('').to_f
  end
end<commit_msg>new method for comparing old global stats to new<commit_after>module Wdpa::GlobalStatsImporter
  extend self

  GLOBAL_STATS_CSV = Rails.root.join('lib/data/seeds/global_stats.csv').freeze

  EXCLUDED_ATTRS = %w(id created_at updated_at).freeze

  def self.import
    attrs = {singleton_guard: 0}
    CSV.foreach(GLOBAL_STATS_CSV, headers: true) do |row|
      field = row['type']
      value = parse_value(row['value'])
      attrs.merge!("#{field}": value)
    end

    stats = GlobalStatistic.new(attrs)
    return stats.save if first_instance.nil?

    # Destroy GlobalStatistic if it differs and recreate
    if first_instance.attributes.except!(*EXCLUDED_ATTRS) != stats.attributes.except!(*EXCLUDED_ATTRS) 
      GlobalStatistic.destroy_all && stats.save
    end
  end

  private

  # If it's a string, ensure to remove commas before casting to float.
  # If it's a float this will basically return the value as it is in the csv.
  # Even though strings in the csv are mostly integers, converting it to float here
  # shouldn't cause issues with the database where the field is explicitly an integer.
  # Postgres should take care of it.
  def self.parse_value(val)
    val.to_s.split(',').join('').to_f
  end

  def first_instance
    GlobalStatistic.first
  end
end
{'fraction_non_alphanumeric': 0.0907840440165062, 'fraction_numerical': 0.0009170105456212746, 'mean_word_length': 3.632696390658174, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4542557', 'n_tokens_mistral': 708, 'n_tokens_neox': 675, 'n_words': 218}