<commit_before>require 'csv'
require 'active_fedora'

namespace :tufts_data do

  task :disk_usage_report, [:arg1] => :environment do |t, args|
    if args[:arg1].nil?
      puts "YOU MUST SPECIFY FULL PATH TO FILE, ABORTING!"
      next
    end
    out = CSV.open("file.csv", "a+")
    CSV.foreach(args[:arg1], encoding: "ISO8859-1") do |row|
      pid = row[0]
      begin
        record = TuftsBase.find(pid, cast: true)
      rescue ActiveFedora::ObjectNotFoundError
        puts "ERROR Could not locate object: #{pid}"
        next
      end
      if record.kind_of?(Array)
        puts "ERROR Multiple results for: #{pid}"
        next
      end

      begin
        collection = nil
        collection = record.object_relations[:is_member_of_collection].first if record.object_relations[:is_member_of_collection]
        case record.class.to_s
          when "TuftsPdf", "TuftsEAD", "TuftsTEI"
            size = File.size record.local_path_for 'Archival.pdf'
            out << [pid,record.class,record.steward.first,collection,(size.to_f / 2**20).round(2)]
          when "TuftsAudio"
            size = File.size record.local_path_for 'ARCHIVAL_WAV'
            out << [pid,record.class,record.steward.first,collection,(size.to_f / 2**20).round(2)]
            size = File.size record.local_path_for 'ARCHIVAL_XML'
            out << [pid,record.class,record.steward.first,collection,(size.to_f / 2**20).round(2)]
#          when "TuftsGenericObject"
#            size = File.size record.local_path_for 'GENERIC-CONTENT'
#            out << [pid,record.class,record.steward.first,(size.to_f / 2**20).round(2)]
          when 'TuftsImage'
            size = File.size record.local_path_for 'Archival.tif'
            out << [pid,record.class,record.steward.first,collection,(size.to_f / 2**20).round(2)]
          when 'TuftsRCR'
            size = File.size record.local_path_for 'RCR-CONTENT'
            out << [pid,record.class,record.steward.first,collection,(size.to_f / 2**20).round(2)]
          when 'TuftsVideo'
            size = File.size record.local_path_for 'Archival.video'
            out << [pid,record.class,record.steward.first,collection,(size.to_f / 2**20).round(2)]
            size = File.size record.local_path_for 'ARCHIVAL_XML'
            out << [pid,record.class,record.steward.first,collection,(size.to_f / 2**20).round(2)]
          when 'TuftsVotingRecord'
            size = File.size record.local_path_for 'RECORD-XML'
            out << [pid,record.class,record.steward.first,collection,(size.to_f / 2**20).round(2)]
          else
            puts "#{record.class} for #{pid} unknown"
          end
          
      rescue => exception
        puts "ERROR There was an error collecting data for: #{pid}"
        puts exception.backtrace
        next
      end
    end 
  end
end<commit_msg>fallback for collection, and add creatordept<commit_after>require 'csv'
require 'active_fedora'

namespace :tufts_data do

  task :disk_usage_report, [:arg1] => :environment do |t, args|
    if args[:arg1].nil?
      puts "YOU MUST SPECIFY FULL PATH TO FILE, ABORTING!"
      next
    end
    out = CSV.open("file.csv", "a+")
    CSV.foreach(args[:arg1], encoding: "ISO8859-1") do |row|
      pid = row[0]
      begin
        record = TuftsBase.find(pid, cast: true)
      rescue ActiveFedora::ObjectNotFoundError
        puts "ERROR Could not locate object: #{pid}"
        next
      end
      if record.kind_of?(Array)
        puts "ERROR Multiple results for: #{pid}"
        next
      end

      begin
        collection = nil
        if record.object_relations[:is_member_of_collection] 
          collection = record.object_relations[:is_member_of_collection].first
        elsif record.object_relations[:has_description] 
          collection = record.object_relations[:has_description].first
        end
        case record.class.to_s
          when "TuftsPdf", "TuftsEAD", "TuftsTEI"
            size = File.size record.local_path_for 'Archival.pdf'
            out << [pid,record.class,record.steward.first,collection,record.creatordept,(size.to_f / 2**20).round(2)]
          when "TuftsAudio"
            size = File.size record.local_path_for 'ARCHIVAL_WAV'
            out << [pid,record.class,record.steward.first,collection,record.creatordept,(size.to_f / 2**20).round(2)]
            size = File.size record.local_path_for 'ARCHIVAL_XML'
            out << [pid,record.class,record.steward.first,collection,record.creatordept,(size.to_f / 2**20).round(2)]
#          when "TuftsGenericObject"
#            size = File.size record.local_path_for 'GENERIC-CONTENT'
#            out << [pid,record.class,record.steward.first,(size.to_f / 2**20).round(2)]
          when 'TuftsImage'
            size = File.size record.local_path_for 'Archival.tif'
            out << [pid,record.class,record.steward.first,collection,record.creatordept,(size.to_f / 2**20).round(2)]
          when 'TuftsRCR'
            size = File.size record.local_path_for 'RCR-CONTENT'
            out << [pid,record.class,record.steward.first,collection,record.creatordept,(size.to_f / 2**20).round(2)]
          when 'TuftsVideo'
            size = File.size record.local_path_for 'Archival.video'
            out << [pid,record.class,record.steward.first,collection,record.creatordept,(size.to_f / 2**20).round(2)]
            size = File.size record.local_path_for 'ARCHIVAL_XML'
            out << [pid,record.class,record.steward.first,collection,record.creatordept,(size.to_f / 2**20).round(2)]
          when 'TuftsVotingRecord'
            size = File.size record.local_path_for 'RECORD-XML'
            out << [pid,record.class,record.steward.first,collection,record.creatordept,(size.to_f / 2**20).round(2)]
          else
            puts "#{record.class} for #{pid} unknown"
          end
          
      rescue => exception
        puts "ERROR There was an error collecting data for: #{pid}"
        puts exception.backtrace
        next
      end
    end 
  end
end
{'fraction_non_alphanumeric': 0.1254163890739507, 'fraction_numerical': 0.014990006662225183, 'mean_word_length': 2.5595732068761112, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4407927', 'n_tokens_mistral': 2004, 'n_tokens_neox': 1927, 'n_words': 363}