<commit_before>namespace :sync do
  task :countries => :environment do
    response = HTTParty.get('http://speciesplus.net/api/v1/geo_entities?geo_entity_types_set=3')
    unless response.code == 200
      Rails.logger.info "Something went wrong while fetching countries"
      Rails.logger.info "Message: #{response.message}"
    end
    geo_entities = response["geo_entities"]
    geo_entities.each do |geo_entity|
      if geo_entity["geo_entity_type"] == "COUNTRY"
        id = geo_entity["id"]
        name = geo_entity["name"]
        iso_code2 = geo_entity["iso_code2"]
        country = Country.find_by_id(id)
        if country.present?
          updated = false
          if name != country.name
            country.update_attributes(name: name)
            updated = true
          elsif iso_code2 != country.iso_code2
            country.update_attributes(iso_code2: iso_code2) unless iso_code2 != country.iso_code2
            updated = true
          end
          Rails.logger.info "Country with ID: #{id} has been updated!" if updated
        else
          Country.create({id: id, name: geo_entity["name"], iso_code2: geo_entity["iso_code2"]})
          Rails.logger.info "Country with name: #{name} has been created!"
        end
      end
    end
  end
end<commit_msg>simplified sync task and added logic to delete countries<commit_after>namespace :sync do
  task :countries => :environment do
    response = HTTParty.get('http://speciesplus.net/api/v1/geo_entities?geo_entity_types_set=3')
    unless response.code == 200
      Rails.logger.info 'Something went wrong while fetching countries from Species+'
      Rails.logger.info 'Message: #{response.message}'
    end
    geo_entities = response["geo_entities"]
    countries = geo_entities.select!{ |ge| ge['geo_entity_type'] == 'COUNTRY' }
    countries.each do |geo_entity|
      id = geo_entity['id']
      name = geo_entity['name']
      iso_code2 = geo_entity['iso_code2']
      country = Country.find_by_id(id)
      status = nil
      if country.present?
        country.assign_attributes(name: name, iso_code2: iso_code2)
        status = 'UPDATED' if country.changed?
        country.save
      else
        status = 'INSERTED'
        Country.create({
          id: id, name: name, iso_code2: iso_code2
        })
      end
      Rails.logger.info "#{status} country #{name} #{iso_code2}(#{id})" if status
    end
    ids = countries.map{ |c| c['id'] }
    (Country.pluck(:id) - ids).each do |id_to_delete|
      country = Country.find(id_to_delete)
      status = 'DELETED'
      unless country.organisations.empty?
        status = 'FAILED TO DELETE'
      end
      Rails.logger.warn "#{status} country #{country.name} #{country.iso_code2}(#{country.id})"
    end
  end
end
{'fraction_non_alphanumeric': 0.11376811594202899, 'fraction_numerical': 0.010144927536231883, 'mean_word_length': 2.787379972565158, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5423426', 'n_tokens_mistral': 878, 'n_tokens_neox': 857, 'n_words': 200}