<commit_before>#!/bin/bash

for i in `ls $1 | tail`;
  do
  echo $i
  find sgf19/$i/ -name "*.sgf" | wc -l;
  echo -en "B+\t\t"
  find sgf19/$i/ -name "*.sgf" -print0 | xargs -0 grep "B+" | wc -l
  echo -en "W+\t\t"
  find sgf19/$i/ -name "*.sgf" -print0 | xargs -0 grep "W+" | wc -l
  echo -en "B+Resign\t"
  find sgf19/$i/ -name "*.sgf" -print0 | xargs -0 grep "B+R" | wc -l
  echo -en "W+Resign\t"
  find sgf19/$i/ -name "*.sgf" -print0 | xargs -0 grep "W+R" | wc -l
  #echo "Stats:"
  #find sgf19/$i/ -name "*.sgf" -exec /bin/sh -c 'tr -cd \; < {} | wc -c' \; | ministat -n 
  echo -en "\n"
done;
<commit_msg>add the right flags to make grep break on first match.  makes it much faster.<commit_after>#!/bin/bash

for i in `ls $1 | tail -n 5`;
  do
  echo $i
  find sgf19/$i/ -name "*.sgf" | wc -l;
  echo -en "B+\t\t"
  find sgf19/$i/ -name "*.sgf" -print0 | xargs -0 grep -m 1 "B+" | wc -l
  echo -en "W+\t\t"
  find sgf19/$i/ -name "*.sgf" -print0 | xargs -0 grep -m 1 "W+" | wc -l
  echo -en "B+Resign\t"
  find sgf19/$i/ -name "*.sgf" -print0 | xargs -0 grep -m 1 "B+R" | wc -l
  echo -en "W+Resign\t"
  find sgf19/$i/ -name "*.sgf" -print0 | xargs -0 grep -m 1 "W+R" | wc -l
  #echo "Stats:"
  #find sgf19/$i/ -name "*.sgf" -exec /bin/sh -c 'tr -cd \; < {} | wc -c' \; | ministat -n 
  echo -en "\n"
done;

{'fraction_non_alphanumeric': 0.22737642585551332, 'fraction_numerical': 0.03574144486692015, 'mean_word_length': 2.081967213114754, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5060390', 'n_tokens_mistral': 686, 'n_tokens_neox': 641, 'n_words': 206}