<commit_before>#encoding: utf-8

namespace :import do

  desc 'Fix symbols in the data files'
  task :fix_symbols => :environment do
    Sapi.disable_triggers
    CSV.foreach('lib/assets/files/ascii_symbols_utf8.csv', :quote_char => 'h') do |row|
      puts "############################# Cleaning References  of ( #{row[0]} => #{row[1]}) #############################"
      count = Reference.where('citation Like ?', "%#{row[0]}%").count
      puts "#{count} records affected"
      Reference.where('citation Like ?', "%#{row[0]}%").each do |o|
        o.citation = o.citation.gsub(row[0], row[1])
        o.save
      end
      puts "############################# Cleaning Geo Entityes  of ( #{row[0]} => #{row[1]})  #############################"
      count = GeoEntity.where('name_en Like ?', "%#{row[0]}%").count
      puts "#{count} records affected"
      GeoEntity.where('name_en Like ?', "%#{row[0]}%").each do |o|
        o.name_en = o.name_en.gsub(row[0], row[1])
        o.save
      end
      puts "############################# Cleaning Taxon Concept of ( #{row[0]} => #{row[1]}) #############################"
      count = TaxonConcept.where('author_year Like ?', "%#{row[0]}%").count
      puts "#{count} records affected"
      TaxonConcept.where('author_year Like ?', "%#{row[0]}%").each do |o|
        o.author_year = o.author_year.gsub(row[0], row[1])
        o.save
      end
      puts "############################# Cleaning Common Names  of ( #{row[0]} => #{row[1]}) #############################"
      count = CommonName.where('name Like ?', "%#{row[0]}%").count
      puts "#{count} records affected"
      CommonName.where('name Like ?', "%#{row[0]}%").each do |o|
        o.name = o.name.gsub(row[0], row[1])
        o.save
      end
    end
    puts "Adding italics into references"
    puts "#{Reference.where('citation like ?', "_%_").count} records affected"
    Reference.where('citation like ? ', "_%_").each do |o|
      o.citation = o.citation.gsub(/_([^_]+)_/, "<i>\\1</i>")
      o.save
    end
    Sapi.enable_triggers
  end<commit_msg>strip spaces<commit_after>#encoding: utf-8

namespace :import do

  desc 'Fix symbols in the data files'
  task :fix_symbols => :environment do
    Sapi.disable_triggers
    CSV.foreach('lib/assets/files/ascii_symbols_utf8.csv', :quote_char => 'h') do |row|
      puts "############################# Cleaning References  of ( #{row[0]} => #{row[1]}) #############################"
      count = Reference.where('citation Like ?', "%#{row[0]}%").count
      puts "#{count} records affected"
      Reference.where('citation Like ?', "%#{row[0]}%").each do |o|
        o.citation = o.citation.gsub(row[0], row[1].strip)
        o.save
      end
      puts "############################# Cleaning Geo Entityes  of ( #{row[0]} => #{row[1]})  #############################"
      count = GeoEntity.where('name_en Like ?', "%#{row[0]}%").count
      puts "#{count} records affected"
      GeoEntity.where('name_en Like ?', "%#{row[0]}%").each do |o|
        o.name_en = o.name_en.gsub(row[0], row[1].strip)
        o.save
      end
      puts "############################# Cleaning Taxon Concept of ( #{row[0]} => #{row[1]}) #############################"
      count = TaxonConcept.where('author_year Like ?', "%#{row[0]}%").count
      puts "#{count} records affected"
      TaxonConcept.where('author_year Like ?', "%#{row[0]}%").each do |o|
        o.author_year = o.author_year.gsub(row[0], row[1].strip)
        o.save
      end
      puts "############################# Cleaning Common Names  of ( #{row[0]} => #{row[1]}) #############################"
      count = CommonName.where('name Like ?', "%#{row[0]}%").count
      puts "#{count} records affected"
      CommonName.where('name Like ?', "%#{row[0]}%").each do |o|
        o.name = o.name.gsub(row[0], row[1].strip)
        o.save
      end
    end
    puts "Adding italics into references"
    puts "#{Reference.where('citation like ?', "_%_").count} records affected"
    Reference.where('citation like ? ', "_%_").each do |o|
      o.citation = o.citation.gsub(/_([^_]+)_/, "<i>\\1</i>")
      o.save
    end
    Sapi.enable_triggers
  end
{'fraction_non_alphanumeric': 0.28796938531451804, 'fraction_numerical': 0.012915570437694332, 'mean_word_length': 3.3744769874476988, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6313714', 'n_tokens_mistral': 1394, 'n_tokens_neox': 1389, 'n_words': 290}