<commit_before>#!/bin/bash

set -e

source ./.checker/scripts/set-env.sh

home_dir=`pwd ./`
jar_dir="jar/"


# download stanford corenlp
url=http://nlp.stanford.edu/software/stanford-corenlp-full-2018-02-27.zip
zip=stanford-corenlp-full-2018-02-27.zip
dir=stanford-corenlp-full-2018-02-27
file=stanford-corenlp-3.9.1.jar
file_model=stanford-corenlp-3.9.1-models.jar

# download Stanford CoreNLP models
wget ${url}

# unpack
unzip ${zip}

cp ${dir}"/"${file} ${jar_dir}
cp ${dir}"/"${file_model} ${jar_dir}


# create jigg jar file
cd ${home_dir}
jigg_file="target/jigg-assembly-0.7.2.jar"
./bin/sbt assembly
cp ${jigg_file} ${jar_dir}
<commit_msg>change version number in checker<commit_after>#!/bin/bash

set -e

source ./.checker/scripts/set-env.sh

home_dir=`pwd ./`
jar_dir="jar/"


# download stanford corenlp
url=http://nlp.stanford.edu/software/stanford-corenlp-full-2018-02-27.zip
zip=stanford-corenlp-full-2018-02-27.zip
dir=stanford-corenlp-full-2018-02-27
file=stanford-corenlp-3.9.1.jar
file_model=stanford-corenlp-3.9.1-models.jar

# download Stanford CoreNLP models
wget ${url}

# unpack
unzip ${zip}

cp ${dir}"/"${file} ${jar_dir}
cp ${dir}"/"${file_model} ${jar_dir}


# create jigg jar file
cd ${home_dir}
jigg_file="target/jigg-assembly-0.8.0.jar"
./bin/sbt assembly
cp ${jigg_file} ${jar_dir}

{'fraction_non_alphanumeric': 0.18126428027418126, 'fraction_numerical': 0.05026656511805026, 'mean_word_length': 4.591489361702128, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6487963', 'n_tokens_mistral': 604, 'n_tokens_neox': 561, 'n_words': 88}