<commit_before>class Dictionary
  DICTIONARIES = {
    en: "#{Rails.root}/db/cmudict.0.7a.txt"
  }
  
  def self.generate(language=:en)
    IO.foreach(DICTIONARIES[language]).each do |line|
      next if line !~ /^[A-Z]/
      
      line.chomp!

      (word, *phonemes) = line.split(/  ?/)

      # Ignore alternative pronunciations
      # 
      next if word =~ /\(\d\) ?$/
      
      # Count the number of stressed vowels
      # 
      syllables = phonemes.grep(/^[AEIUO]/).size
      
      create(word, phonemes, syllables, language)
    end
  end
  
  def self.create(word, phonemes, syllables, language=:en)
    puts "#{word} #{phonemes} - #{syllables} (#{language})"

    Word.create!(word: word.upcase, phonemes: phonemes, syllables: syllables.to_i, language: language)
  end
end<commit_msg>Adds comment<commit_after>class Dictionary
  DICTIONARIES = {
    en: "#{Rails.root}/db/cmudict.0.7a.txt"
  }
  
  def self.generate(language=:en)
    IO.foreach(DICTIONARIES[language]).each do |line|
      next if line !~ /^[A-Z]/
      
      line.chomp!

      (word, *phonemes) = line.split(/  ?/)

      # Ignore alternative pronunciations
      # 
      next if word =~ /\(\d\) ?$/
      
      # Count the number of stressed vowels
      # 
      syllables = phonemes.grep(/^[AEIUO]/).size
      
      create(word, phonemes, syllables, language)
    end
  end
  
  def self.create(word, phonemes, syllables, language=:en)
    puts "#{word} #{phonemes} - #{syllables} (#{language})"

    Word.create!(word: word.upcase, phonemes: phonemes, syllables: syllables.to_i, language: language)
  end
end # Dictionary
{'fraction_non_alphanumeric': 0.14876543209876544, 'fraction_numerical': 0.0024691358024691358, 'mean_word_length': 2.539301310043668, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7174680', 'n_tokens_mistral': 623, 'n_tokens_neox': 578, 'n_words': 119}