<commit_before>#!/bin/bash

java  -classpath ${CLASSPATH}:${JOSHUA}/bin \
      -Djava.library.path=${JOSHUA}/lib \
      -Dfile.encoding=utf8 \
      -Djava.util.logging.config.file=${JOSHUA}/logging.properties \
      -Xmx55g -Xms55g \
      joshua.decoder.JoshuaDecoder parse.config <$INPUT 2>parse.log<commit_msg>modified example parse script<commit_after>#!/bin/bash

java  -classpath ${CLASSPATH}:${JOSHUA}/bin \
      -Djava.library.path=${JOSHUA}/lib \
      -Dfile.encoding=utf8 \
      -Djava.util.logging.config.file=${JOSHUA}/logging.properties \
      -Xmx55g -Xms55g \
      joshua.decoder.JoshuaDecoder $CONF <$INPUT 2>$LOG
{'fraction_non_alphanumeric': 0.16300940438871472, 'fraction_numerical': 0.018808777429467086, 'mean_word_length': 3.0961538461538463, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1039821', 'n_tokens_mistral': 252, 'n_tokens_neox': 240, 'n_words': 28}