<commit_before>require 'csv'

namespace :csv do

  task :import, [:filename, :username] => [:environment] do |t, args|
    user = User.find_by_name(args.username)
    csv_text = File.read(args.filename)
    csv = CSV.parse(csv_text, :headers => true)
    csv.each do |row|
      row = row.to_hash.with_indifferent_access
      Entry.create!(row.to_hash.symbolize_keys.merge(:user => user))
    end
  end<commit_msg>handle duplicate entries in csv import rake task.<commit_after>require 'csv'

namespace :csv do

  # import glossary entries from a csv file
  #
  # create a user to add the entries and specify the user in the username variable, as in:
  # rake csv:import[csv-file,importer]
  # for a user named "importer"
  #
  # first line is a header and should include term_in_<lang> for each language in the glossary
  # plus optionally "note" for translator notes
  # entries with duplicate terms in the glossary language are entered with an index
  task :import, [:filename, :username] => [:environment] do |t, args|
    user = User.find_by_name(args.username)
    csv_text = File.read(args.filename)
    csv = CSV.parse(csv_text, :headers => true)
    csv.each do |row|
      row = row.to_hash.with_indifferent_access
      entry = Entry.new(row.to_hash.symbolize_keys.merge(:user => user))
      term = entry.term_in_glossary_language
      i = 0
      while (Entry.find_by_term_in_glossary_language(entry.term_in_glossary_language)) 
        entry.term_in_glossary_language = term + (i += 1).to_s
      end
      if entry.save
        puts "Entry '#{entry.term_in_glossary_language}' saved."
      else
        puts "Entry save failed for '#{term}'."
      end
    end
  end
{'fraction_non_alphanumeric': 0.1021984551396316, 'fraction_numerical': 0.0011883541295306002, 'mean_word_length': 3.4083769633507854, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1259082', 'n_tokens_mistral': 562, 'n_tokens_neox': 542, 'n_words': 157}