<commit_before>#!/usr/bin/env bash
source_path=$1
target_path=$2
inter_dir="inter"
model="basic"
run_id="01"
step="015000"
debug="False"
python3 -m squad.prepro_simple --mode single --single_path $source_path --debug $debug --target_dir $inter_dir
answer_path="$inter_dir/$model/$run_id/answer/single-$step.json"
python3 -m basic.cli --run_id $run_id --out_base_dir $inter_dir --model $model --load_step $step --draft $debug --eval_num_batches 0 --mode forward --attention --batch_size 8 --use_glove_for_unk --known_if_glove --nofinetune --notraditional<commit_msg>coda lab match for run.sh<commit_after>#!/usr/bin/env bash
source_path=$1
target_path=$2
inter_dir="inter"
model="basic"
run_id="01"
step="015000"
debug="False"
python3 -m squad.prepro_simple --mode single --single_path $source_path --debug $debug --target_dir $inter_dir --glove_dir .
answer_path="$inter_dir/$model/$run_id/answer/single-$step.json"
python3 -m basic.cli --run_id $run_id --out_base_dir $inter_dir --model $model --load_step $step --draft $debug --eval_num_batches 0 --mode forward --attention --batch_size 8 --use_glove_for_unk --known_if_glove --nofinetune --notraditional
{'fraction_non_alphanumeric': 0.1522491349480969, 'fraction_numerical': 0.02422145328719723, 'mean_word_length': 4.994818652849741, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6654405', 'n_tokens_mistral': 430, 'n_tokens_neox': 409, 'n_words': 94}