<commit_before>#require 'uri'
namespace :iucn do

  desc 'Update mapping between CITES species and IUCN species'
  task :mapping => :environment do
    API_URL = 'http://rlapiv3-beta.iucnredlist.org/api/v3/species/'

    config_location = Rails.root.join('config/secrets.yml')
    config = YAML.load_file(config_location)[Rails.env]
    token = config['iucn_redlist']

    rank = Rank.where(:name => Rank::SPECIES).first
    taxonomy = Taxonomy.where(:name => Taxonomy::CITES_EU).first

    TaxonConcept.where(:rank_id => rank.id, :name_status => 'A',
                       :taxonomy_id => taxonomy.id).limit(200).each do |tc|
      url = URI.escape("#{API_URL}#{tc.full_name.downcase}?token=#{token}")
      data = JSON.parse(RestClient.get(url))
      begin
        unless data["result"].empty?
          puts "#{tc.full_name} #{tc.author_year} <=>  #{data["name"]} #{data["result"].first["authority"]}"
        else
          puts "#{tc.full_name} NO MATCH"
        end
      rescue
        puts "CABOOM: #{data.inspect}"
      end
    end
  end
end<commit_msg>Inserts matches nad non matches into the Database<commit_after>#require 'uri'
namespace :iucn do

  desc 'Update mapping between CITES species and IUCN species'
  task :mapping => :environment do
    API_URL = 'http://rlapiv3-beta.iucnredlist.org/api/v3/species/'

    config_location = Rails.root.join('config/secrets.yml')
    config = YAML.load_file(config_location)[Rails.env]
    token = config['iucn_redlist']

    rank = Rank.where(:name => Rank::SPECIES).first
    taxonomy = Taxonomy.where(:name => Taxonomy::CITES_EU).first

    TaxonConcept.where(:rank_id => rank.id, :name_status => 'A',
                       :taxonomy_id => taxonomy.id).each do |tc|
      map = Admin::IucnMapping.find_or_create_by_taxon_concept_id(tc.id)
      url = URI.escape("#{API_URL}#{tc.full_name.downcase}?token=#{token}")
      data = JSON.parse(RestClient.get(url))
      begin
        unless data["result"].empty?
          result = data["result"].first
          puts "#{tc.full_name} #{tc.author_year} <=>  #{result["scientific_name"]} #{result["authority"]}"
          map.update_attributes(
            :iucn_taxon_name => result['scientific_name'],
            :iucn_taxon_id => result['taxonid'],
            :iucn_author => result['authority'],
            :iucn_category => result['category']
          )
        else
          puts "#{tc.full_name} NO MATCH"
        end
      rescue
        puts "CABOOM: #{data.inspect}"
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.1489277204130262, 'fraction_numerical': 0.0027799841143764893, 'mean_word_length': 2.9359375, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6675888', 'n_tokens_mistral': 863, 'n_tokens_neox': 841, 'n_words': 151}