<commit_before>
import Glibc
import Sua

if !Stdin.isTerminal {

  var stream = CodeUnitStream()
  var bytesCount = 0
  var newLineCount = 0

  try Stdin.readByteLines() { line in
    let len = line.count
    bytesCount += len
    if len > 0 && line[len - 1] == 10 {
      newLineCount += 1
    }
    p(line)
    p(String.fromCharCodes(line))
    //stream.codeUnits = line
  }

  print("  \(newLineCount)  ( - )   \(bytesCount)")
} else {
  print("Usage: call it by passing a pipe or standard input.\n" +
    "E.g. WordCounter < some_sample.txt\n" +
    "     ls -l | WordCounter")
}<commit_msg>Added actual word counting support to the word counter example.<commit_after>
import Glibc
import Sua

if !Stdin.isTerminal {

  var stream = CodeUnitStream()
  var bytesCount = 0
  var newLineCount = 0
  var wordCount = 0

  try Stdin.readByteLines() { line in
    let len = line.count
    bytesCount += len
    if len > 0 && line[len - 1] == 10 {
      newLineCount += 1
    }
    //p(line)
    //p(String.fromCharCodes(line))
    stream.codeUnits = line
    while !stream.isEol {
      // 32 - space; 10 - new line; 9 - tab; 13 - carriage return
      // 160 - unicode space: \u00a0
      stream.eatWhile { (c: UInt8) in
        return c == 32 || c == 10 || c == 9 || c == 13 || c == 160
      }
      if stream.eatWhileNeitherFive(32, c2: 10, c3: 9, c4: 13, c5: 160) {
        wordCount += 1
      }
    }
  }

  print("  \(newLineCount)   \(wordCount)   \(bytesCount)")
} else {
  print("Usage: call it by passing a pipe or standard input.\n" +
    "E.g. WordCounter < some_sample.txt\n" +
    "     ls -l | WordCounter")
}
{'fraction_non_alphanumeric': 0.13493530499075784, 'fraction_numerical': 0.033271719038817, 'mean_word_length': 2.2808080808080806, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4319517', 'n_tokens_mistral': 628, 'n_tokens_neox': 569, 'n_words': 170}