<commit_before>module Stats::CountryStatisticsImporter
  def self.import
    import_stats(stats_csv_path, CountryStatistic)
    import_stats(pame_csv_path,  PameStatistic)

    # Import stats from DOPA services
    Stats::CountryStatisticsApi.import

    # Import Aichi11Target stats from CSV
    Aichi11Target.instance
  end

  def self.import_stats path, model
    countries = Country.pluck(:id, :iso_3).each_with_object({}) { |(id, iso_3), hash|
      hash[iso_3] = id
    }

    CSV.foreach(path, headers: true) do |row|
      country_iso3 = row.delete('iso3').last
      country_id = countries[country_iso3]
      # If the value is na (not applicable) use nil
      row.each { |key, value| row[key] = nil if value && value.downcase == 'na' }
      attrs = {country_id: country_id}.merge(row)
      attrs = attrs.merge(pame_assessments(country_id)) if model == PameStatistic

      model.create(attrs)
    end
  end

  def self.pame_assessments(country_id)
    return {} unless country_id

    country = Country.find(country_id)

    {
      assessments: country.assessments,
      assessed_pas: country.assessed_pas
    }
  end

  def self.stats_csv_path
    Rails.root.join('lib/data/seeds/country_statistics.csv')
  end

  def self.pame_csv_path
    Rails.root.join('lib/data/seeds/pame_country_stats.csv')
  end
end<commit_msg>revert back to using countries oecm path - just for the aichi11 target dashboard stats<commit_after>module Stats::CountryStatisticsImporter
  def self.import
    import_stats(stats_csv_path, CountryStatistic)
    import_stats(pame_csv_path,  PameStatistic)

    # Import stats from DOPA services
    Stats::CountryStatisticsApi.import

    # Import Aichi11Target stats from CSV
    Aichi11Target.instance
  end

  def self.import_stats path, model
    countries = Country.pluck(:id, :iso_3).each_with_object({}) { |(id, iso_3), hash|
      hash[iso_3] = id
    }

    CSV.foreach(path, headers: true) do |row|
      country_iso3 = row.delete('iso3').last
      country_id = countries[country_iso3]
      # If the value is na (not applicable) use nil
      row.each { |key, value| row[key] = nil if value && value.downcase == 'na' }
      attrs = {country_id: country_id}.merge(row)
      attrs = attrs.merge(pame_assessments(country_id)) if model == PameStatistic

      model.create(attrs)
    end
  end

  def self.pame_assessments(country_id)
    return {} unless country_id

    country = Country.find(country_id)

    {
      assessments: country.assessments,
      assessed_pas: country.assessed_pas
    }
  end

  def self.stats_csv_path
    Rails.root.join('lib/data/seeds/country_statistics_oecm.csv')
  end

  def self.pame_csv_path
    Rails.root.join('lib/data/seeds/pame_country_stats.csv')
  end
end
{'fraction_non_alphanumeric': 0.09861717612809316, 'fraction_numerical': 0.008005822416302766, 'mean_word_length': 3.477198697068404, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1684329', 'n_tokens_mistral': 950, 'n_tokens_neox': 921, 'n_words': 196}