<commit_before>package utils

import scala.collection.immutable.ListMap
import play.api.libs.iteratee._

object WordCount {

  // count words using provided wordMap so that computation doesn't start from scratch
  
  /** Counts words in string, using specified Map[String, Int] as accumulator.
   *  @param    s String with words. Case-insensitive, ignores non-letters
   *  @param    wordMap Map[String, Int] with word counts
   *  @return   Map[String, Int] with word counts
   */
  def countWords(s: String, wordMap: Map[String, Int]): Map[String, Int] =
    s.replaceAll("[^a-zA-Z ]", "").split(" ").foldLeft(wordMap) {
      case (acc, el) => acc + ((el.toLowerCase, acc.getOrElse(el.toLowerCase, 0) + 1))
    }

  /** Generates ListMap with Top n most popular words in wordmap
   *  @param    wordMap Map[String, Int] with word counts
   *  @param    n number highest ranking words to return
   *  @return   sorted ListMap with top n words in descending order of count 
   */
  def topN(wordMap: Map[String, Int], n: Int): ListMap[String, Int] =
    ListMap[String, Int](wordMap.toList.sortBy(_._2).reverse.take(n): _*)

  /** Generate string from TimeInterval for n significant Interval components (e.g. days and hours)
   *  @param    n number of siginificant interval components to print
   *  @return   String representation of TimeInterval
   */
  def wordCountIteratee(f: Map[String, Int] => Unit) =
    Iteratee.fold[String, Map[String, Int]](Map[String, Int]()) {
      case (acc, el) => {
        val newAcc = countWords(el, acc)
        println(topN(newAcc, 5))
        f(newAcc) // purely for side-effects such as println, pushing to websocket / eventStream or testing
        newAcc
      }
    }

}<commit_msg>documentation, remove println in iteratee<commit_after>package utils

import scala.collection.immutable.ListMap
import play.api.libs.iteratee._

object WordCount {

  // count words using provided wordMap so that computation doesn't start from scratch
  
  /** Counts words in string, using specified Map[String, Int] as accumulator.
   *  @param    s String with words. Case-insensitive, ignores non-letters
   *  @param    wordMap Map[String, Int] with word counts
   *  @return   Map[String, Int] with word counts
   */
  def countWords(s: String, wordMap: Map[String, Int]): Map[String, Int] =
    s.replaceAll("[^a-zA-Z ]", "").split(" ").foldLeft(wordMap) {
      case (acc, el) => acc + ((el.toLowerCase, acc.getOrElse(el.toLowerCase, 0) + 1))
    }

  /** Generates ListMap with Top n most popular words in wordmap
   *  @param    wordMap Map[String, Int] with word counts
   *  @param    n number highest ranking words to return
   *  @return   sorted ListMap with top n words in descending order of count 
   */
  def topN(wordMap: Map[String, Int], n: Int): ListMap[String, Int] =
    ListMap[String, Int](wordMap.toList.sortBy(_._2).reverse.take(n): _*)

  /** Generate string from TimeInterval for n significant Interval components (e.g. days and hours).
   *  Allows passing in a side-effecting function, e.g. for testing or pushing data to websocket
   *  or EventStream.
   *  @param    n number of siginificant interval components to print
   *  @return   String representation of TimeInterval
   */
  def wordCountIteratee(f: Map[String, Int] => Unit) =
    Iteratee.fold[String, Map[String, Int]](Map[String, Int]()) {
      case (acc, el) => {
        val newAcc = countWords(el, acc)
        f(newAcc) // purely for side-effects such as println, pushing to websocket / eventStream or testing
        newAcc
      }
    }

}
{'fraction_non_alphanumeric': 0.11251049538203191, 'fraction_numerical': 0.0019591379792891126, 'mean_word_length': 3.2649164677804294, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4405636', 'n_tokens_mistral': 1129, 'n_tokens_neox': 1097, 'n_words': 370}