<commit_before>require 'csv'
namespace :campus_access do
  desc "load a new campus access file for today"
  task load: :environment do
    if ENV['BIBDATA_ACCESS_DIRECTORY']
      file_part = ENV['BIBDATA_ACCESS_FILE_NAME'] || 'Daily file to the Library_fileshare_authorized to be on campus_completed Learn.xlsx'
      trained_file_part = ENV['BIBDATA_TRAINED_FILE_NAME'] || 'Daily file to the Library_fileshare_Learn only.xlsx'
      file_name = File.join(ENV['BIBDATA_ACCESS_DIRECTORY'], file_part)
      trainedfile_name = File.join(ENV['BIBDATA_ACCESS_DIRECTORY'], trained_file_part)
      additional_id_file = Rails.root.join('additional_campus_access.csv')
      additional_ids = CSV.read(additional_id_file, headers: true).map { |row| row[1] }
      if File.exist?(file_name)
        puts "Reading in the daily access file #{file_name}"<commit_msg>Updating default names to match current output The files were not named exactly the same as the test files that were emailed, when they appeared in the share. This makes the defaults match the files in the share.<commit_after>require 'csv'
namespace :campus_access do
  desc "load a new campus access file for today"
  task load: :environment do
    if ENV['BIBDATA_ACCESS_DIRECTORY']
      file_part = ENV['BIBDATA_ACCESS_FILE_NAME'] || 'Daily file to the Library_fileshare_authorized to be on campus and completed Learn-en.xlsx'
      trained_file_part = ENV['BIBDATA_TRAINED_FILE_NAME'] || 'Daily file to the Library_fileshare_Learn only-en.xlsx'
      file_name = File.join(ENV['BIBDATA_ACCESS_DIRECTORY'], file_part)
      trainedfile_name = File.join(ENV['BIBDATA_ACCESS_DIRECTORY'], trained_file_part)
      additional_id_file = Rails.root.join('additional_campus_access.csv')
      additional_ids = CSV.read(additional_id_file, headers: true).map { |row| row[1] }
      if File.exist?(file_name)
        puts "Reading in the daily access file #{file_name}"
{'fraction_non_alphanumeric': 0.0880667014069828, 'fraction_numerical': 0.0010422094841063053, 'mean_word_length': 4.889570552147239, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6247019', 'n_tokens_mistral': 613, 'n_tokens_neox': 596, 'n_words': 166}