<commit_before>namespace :hathi do
  
  desc "compact hathi overlap report to only include files that overlap"
  task compact_overlap: :environment do
    if ENV['HATHI_OVERLAP'] && ENV['HATHI_OVERLAP_COMPACTED'] 
      Hathi::CompactOverlap.perform
      output_file = ENV['HATHI_OVERLAP_COMPACTED']
      sorted_file = ENV['HATHI_OVERLAP_COMPACTED_SORTED'] || "#{output_file}.sorted"
      `sort -t$'\t' -k 1n #{output_file} > #{sorted_file}`
    end
  end

  desc 'Compact hathi_full file to include only the identifier and the oclc'
  task compact_full: :environment do
    if ENV['HATHI_FULL'] && ENV['OUTPUT_HATHI']
      Hathi::CompactFull.compact_full
      output_file = "#{ENV['OUTPUT_HATHI']}/compact_hathi_full.tsv"
      `sort -t$'\t' -k 2n #{output_file} > #{ENV['OUTPUT_HATHI']}/compact_hathi_full_sorted.tsv`
    end  
  end

  desc 'Combine Hathi_full and Hathi_overlap_compact files'
  task merge: :environment do
     sorted_file1 = ENV['HATHI_OVERLAP_COMPACTED_SORTED']
     sorted_file2 = "#{ENV['OUTPUT_HATHI']}/compact_hathi_full_sorted.tsv"
     hathi_final = "#{ENV['OUTPUT_HATHI']}"
    `join -t$'\t' -1 1 -2 2 #{sorted_file1 sorted_file2} > #{hathi_final}`
  end  
end<commit_msg>Update rake tasks with file name variables<commit_after>namespace :hathi do
  
  desc "compact hathi overlap report to only include files that overlap"
  task compact_overlap: :environment do
    if ENV['HATHI_OVERLAP'] && ENV['HATHI_OVERLAP_COMPACTED'] 
      Hathi::CompactOverlap.perform
      output_file = ENV['HATHI_OVERLAP_COMPACTED']
      sorted_file = ENV['HATHI_OVERLAP_COMPACTED_SORTED'] || "#{output_file}.sorted"
      `sort -t$'\t' -k 1n #{output_file} > #{sorted_file}`
    end
  end

  desc 'Compact hathi_full file to include only the identifier and the oclc'
  task compact_full: :environment do
    if ENV['HATHI_FULL'] && ENV['OUTPUT_HATHI']
      Hathi::CompactFull.compact_full
      output_file = "#{ENV['OUTPUT_HATHI']}/compact_hathi_full.tsv"
      sorted_file = "#{ENV['OUTPUT_HATHI']}/compact_hathi_full_sorted.tsv"
      `sort -t$'\t' -k 2n #{output_file} > #{sorted_file}`
    end  
  end

  desc 'Combine Hathi_full and Hathi_overlap_compact files'
  task merge: :environment do
     sorted_file1 = ENV['HATHI_OVERLAP_COMPACTED_SORTED']
     sorted_file2 = "#{ENV['OUTPUT_HATHI']}/compact_hathi_full_sorted.tsv"
     hathi_final = "#{ENV['OUTPUT_HATHI']}/hathi_final.tsv"
    `join -t$'\t' -1 1 -2 2 #{sorted_file1 sorted_file2} > #{hathi_final}`
  end  
end
{'fraction_non_alphanumeric': 0.12419871794871795, 'fraction_numerical': 0.008012820512820512, 'mean_word_length': 4.290254237288136, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3473008', 'n_tokens_mistral': 1004, 'n_tokens_neox': 962, 'n_words': 191}