<commit_before>#!/bin/bash

FST="$1"

sort "$FST" > "$FST.sort"

rm "$FST.sort"<commit_msg>FEATURE/MAJOR: Set the bases of the parsing algorithm<commit_after>#!/bin/bash

FST="$1"

sort "$FST" > "$FST.sort"

cat <<EOF
int pos=0;
int total=0;

EOF

while read DEP ARR CHAR WEIGHT ; do
    : ${WEIGHT:=0}

    if [[ $DEP != $PREV_DEP ]]; then
        cat <<EOF
    }

NODE_$DEP :
    pos++;
    switch (token[pos-1]) {
EOF
    fi

    cat <<EOF
    case '$CHAR':
        total += $WEIGHT;
        goto NODE_$ARR;
EOF

    PREV_DEP=$DEP
done < "$FST.sort"

rm "$FST.sort"
{'fraction_non_alphanumeric': 0.1813380281690141, 'fraction_numerical': 0.01056338028169014, 'mean_word_length': 2.670967741935484, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1941261', 'n_tokens_mistral': 255, 'n_tokens_neox': 238, 'n_words': 59}