<commit_before>#!/usr/bin/env ruby

require 'erb'
require 'audibleturk'

#Name of form field in Mechanical Turk assignment form
# containing URL to audio file
url_at_form_field = ARGV[0] || 'audibleturk_url'

home = "#{Dir.home}/Documents/Software/dist/ruby/audibleturk"

puts "Collecting results from Amazon..."
results = Audibleturk::Remote::Result.all_approved(:url_at => url_at_form_field)

puts "Sorting results..."
available_folders = results.collect{|result| result.transcription.title }.uniq.select{|title| Audibleturk::Folder.named(title)}

template = IO.read("#{home}/www/transcript.html.erb") unless available_folders.empty?

available_folders.each do |folder|
  shortname = folder
  folder = Audibleturk::Folder.named(folder)
  filename = {
    'done' => 'transcript.html',
    'working' => 'transcript_in_progress.html'
  }
  next if File.exists?("#{folder.path}/#{filename['done']}")
  transcription = Audibleturk::Transcription.new(shortname, results.select{|result| result.transcription.title == shortname}.collect{|result| result.transcription})
  transcription.each{|chunk| chunk.url = "audio/#{chunk.filename_local}" }
  transcription.subtitle = folder.subtitle
  File.delete("#{folder.path}/#{filename['working']}") if File.exists?("#{folder.path}/#{filename['working']}")
  is_done = (transcription.to_a.length == folder.audio_chunks)
  out_file = is_done ? filename['done'] : filename['working']<commit_msg>Filename hash in at-collect now uses symbols instead of strings for the keys.<commit_after>#!/usr/bin/env ruby

require 'erb'
require 'audibleturk'

#Name of form field in Mechanical Turk assignment form
# containing URL to audio file
url_at_form_field = ARGV[0] || 'audibleturk_url'

home = "#{Dir.home}/Documents/Software/dist/ruby/audibleturk"

puts "Collecting results from Amazon..."
results = Audibleturk::Remote::Result.all_approved(:url_at => url_at_form_field)

puts "Sorting results..."
available_folders = results.collect{|result| result.transcription.title }.uniq.select{|title| Audibleturk::Folder.named(title)}

template = IO.read("#{home}/www/transcript.html.erb") unless available_folders.empty?

available_folders.each do |folder|
  shortname = folder
  folder = Audibleturk::Folder.named(folder)
  filename = {
    :done => 'transcript.html',
    :working => 'transcript_in_progress.html'
  }
  next if File.exists?("#{folder.path}/#{filename[:done]}")
  transcription = Audibleturk::Transcription.new(shortname, results.select{|result| result.transcription.title == shortname}.collect{|result| result.transcription})
  transcription.each{|chunk| chunk.url = "audio/#{chunk.filename_local}" }
  transcription.subtitle = folder.subtitle
  File.delete("#{folder.path}/#{filename[:working]}") if File.exists?("#{folder.path}/#{filename[:working]}")
  is_done = (transcription.to_a.length == folder.audio_chunks)
  out_file = is_done ? filename[:done] : filename[:working]
{'fraction_non_alphanumeric': 0.15403087478559177, 'fraction_numerical': 0.0006861063464837049, 'mean_word_length': 6.094890510948905, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5124140', 'n_tokens_mistral': 963, 'n_tokens_neox': 925, 'n_words': 173}