<commit_before>#!/bin/bash

project_path=$(python get_path.py)

source $MODULESHOME/init/bash
module load LANG/PYTHON/2.7.6-SYSTEM

export PYTHONPATH=$PYTHONPATH:/cs/natlang-projects/glm-parser/Cython-0.20.1

cd $project_path

echo "Compile Cython classes ..."
python setup.py build_ext --inplace


echo "Compile hvector ..."
cd hvector
python setup.py install --install-lib .


cd ..
python glm_parser.py -i 1 -b 2 -e 2 -t 0,1,22,24 -p ~/Documents/penn-wsj-deps/
<commit_msg>Update train.sh<commit_after>#!/bin/bash

project_path=$(python get_path.py)

source $MODULESHOME/init/bash
module load LANG/PYTHON/2.7.6-SYSTEM

export PYTHONPATH=$PYTHONPATH:/cs/natlang-projects/glm-parser/Cython-0.20.1

cd $project_path

echo "Compile Cython classes ..."
python setup.py build_ext --inplace


echo "Compile hvector ..."
cd hvector
python setup.py install --install-lib .


cd ..
python glm_parser.py -i 1 -b 2 -e 21 -t 0,1,22,24 -p /cs/natlang-projects/glm-parser/penn-wsj-deps/

{'fraction_non_alphanumeric': 0.14358974358974358, 'fraction_numerical': 0.033846153846153845, 'mean_word_length': 3.6698564593301435, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3778421', 'n_tokens_mistral': 419, 'n_tokens_neox': 382, 'n_words': 85}