<commit_before>
def check_file_provided(task_name)
  if ENV['FILE'].blank?
    fail "Usage: FILE=/abs/path/to/file rake elibrary:import:#{task_name}"
  end
end

namespace :elibrary do
  task :clear => :environment do
    puts "Deleting citations"
    DocumentCitationGeoEntity.delete_all
    DocumentCitationTaxonConcept.delete_all
    DocumentCitation.delete_all
    puts "Deleting documents"
    DocumentTag.delete_all
    Document::ProposalDetails.delete_all
    Document::ReviewDetails.delete_all
    Document.delete_all
  end
<commit_msg>task to import document tags<commit_after>
def check_file_provided(task_name)
  if ENV['FILE'].blank?
    fail "Usage: FILE=/abs/path/to/file rake elibrary:import:#{task_name}"
  end
end

namespace :elibrary do
  task :clear => :environment do
    puts "Deleting citations"
    DocumentCitationGeoEntity.delete_all
    DocumentCitationTaxonConcept.delete_all
    DocumentCitation.delete_all
    puts "Deleting documents"
    DocumentTag.delete_all
    Document::ProposalDetails.delete_all
    Document::ReviewDetails.delete_all
    Document.delete_all
  end

  namespace :document_tags do
    desc 'Import missing document tags'
    task :import => :environment do
      puts "#{DocumentTag.count} document tags"

      [
       "I", "II", "III", "IV", "June 1986", "None",
       "Post-CoP11", "Post-CoP12", "Post-CoP13"
      ].each { |tag| DocumentTag::ReviewPhase.find_or_create_by_name(name: tag) }

      [
        "AC review and categorization (k)", "AC review and categorization [k]",
        "AC review (e)", "AC review [e]", "Categorise information (i)", "Consulation (d)",
        "PC review and categorization [k]", "PC review and categorization (m)", "PC review (e)",
        "Research of species [j]", "Selection of species (b)", "Selection of species [b]",
        "Species selection (b)", "Species selection [b]"
      ].each {|tag| DocumentTag::ProcessStage.find_or_create_by_name(name: tag) }

      [
       "Accepted", "Cancelled", "Deferred",
       "Redundant", "Rejected", "Transferred to other proposals",
       "Withdrawn", "Accepted as amended", "Rejected as amended",
       "Adopted"
      ].each { |tag| DocumentTag::ProposalOutcome.find_or_create_by_name(name: tag) }

      puts "#{DocumentTag.count} document tags"
    end
  end

{'fraction_non_alphanumeric': 0.11843817787418655, 'fraction_numerical': 0.004338394793926247, 'mean_word_length': 3.81419624217119, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2950775', 'n_tokens_mistral': 742, 'n_tokens_neox': 716, 'n_words': 183}