<commit_before>REUTERS_DIR=data/reuters21578/
EXTRACT_DIR=/tmp/extracted

if [ ! -d $EXTRACT_DIR ]; then
  mkdir -p $EXTRACT_DIR
fi

ARTICLE_COUNT=0
for file in $REUTERS_DIR/*sgm
do
  EXTRACT_OUTPUT="`./bin/extractor $file $EXTRACT_DIR $ARTICLE_COUNT`"
  ARTICLE_COUNT=$(($ARTICLE_COUNT+${EXTRACT_OUTPUT##*:}))
  echo "Articles: $ARTICLE_COUNT"
done

echo "Total articles: $ARTICLE_COUNT"
<commit_msg>extract-artcles.sh does tokenization and stemming<commit_after>REUTERS_DIR=data/reuters21578/
EXTRACT_DIR=/tmp/extracted
STEM_DIR=/tmp/stemmed

set -e

if [ ! -d $EXTRACT_DIR ]; then
  mkdir -p $EXTRACT_DIR
fi

if [ ! -d $STEM_DIR ]; then
  mkdir -p $STEM_DIR
fi

ARTICLE_COUNT=0
for file in $REUTERS_DIR/*sgm
do
  EXTRACT_OUTPUT="`./bin/extractor $file $EXTRACT_DIR $ARTICLE_COUNT`"
  ARTICLE_COUNT=$(($ARTICLE_COUNT+${EXTRACT_OUTPUT##*:}))
  echo "Articles: $ARTICLE_COUNT"
done
echo "Total articles: $ARTICLE_COUNT"

echo "Tokenizing and stemming"
COUNT=0
for file in `ls $EXTRACT_DIR`
do
	./bin/tokenizer < $EXTRACT_DIR/$file | ./bin/stemmer > $STEM_DIR/$file
#	./bin/tokenizer < $EXTRACT_DIR/$file > $STEM_DIR/$file
	COUNT=$(($COUNT + 1))
	if [ $(($COUNT % 100)) == 0 ]; then
		echo "$COUNT"
	fi
done


{'fraction_non_alphanumeric': 0.1490066225165563, 'fraction_numerical': 0.014900662251655629, 'mean_word_length': 5.168367346938775, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5788927', 'n_tokens_mistral': 528, 'n_tokens_neox': 496, 'n_words': 102}