<commit_before>## Script for commands to tag corpus with TreeTagger

## Path to the directory to save corpus data
DATA=../data
IDIR_EN=$DATA/xml_sentence/EN
IDIR_ES=$DATA/xml_sentence/ES
IDIR_DE=$DATA/xml_sentence/DE
ODIR_EN=$DATA/ttg/EN
ODIR_ES=$DATA/ttg/ES
ODIR_DE=$DATA/ttg/DE

## Create output directories
mkdir -p $ODIR_EN
mkdir -p $ODIR_ES
mkdir -p $ODIR_DE

## English
echo "Tagging English proceedings with TreTagger ...."
for i in $IDIR_EN/*.xml; do echo $i; tree-tagger-english < $i > $ODIR_EN/$(basename $i); done
## Spanish
echo "Tagging Spanish proceedings with TreeTagger ...."
for i in $IDIR_ES/*.xml; do echo $i; tree-tagger-spanish < $i > $ODIR_ES/$(basename $i); done
## German
echo "Tagging German proceedings with TreeTagger ...."
for i in $IDIR_DE/*.xml; do echo $i; tree-tagger-german < $i > $ODIR_DE/$(basename $i); done<commit_msg>clean <unknown> to avoid parsing errors<commit_after>## Script for commands to tag corpus with TreeTagger

## Path to the directory to save corpus data
DATA=../data
IDIR_EN=$DATA/xml_sentence/EN
IDIR_ES=$DATA/xml_sentence/ES
IDIR_DE=$DATA/xml_sentence/DE
ODIR_EN=$DATA/ttg/EN
ODIR_ES=$DATA/ttg/ES
ODIR_DE=$DATA/ttg/DE

## Create output directories
mkdir -p $ODIR_EN
mkdir -p $ODIR_ES
mkdir -p $ODIR_DE

## English
echo "Tagging English proceedings with TreTagger ...."
for i in $IDIR_EN/*.xml; do echo $i; tree-tagger-english < $i > $ODIR_EN/$(basename $i); done
for i in $ODIR_EN/*.xml; do perl ‑CSAD ‑Mutf8 -pi -e 's/<unknown>/@unknown@/g' $i; done
## Spanish
echo "Tagging Spanish proceedings with TreeTagger ...."
for i in $IDIR_ES/*.xml; do echo $i; tree-tagger-spanish < $i > $ODIR_ES/$(basename $i); done
for i in $ODIR_ES/*.xml; do perl ‑CSAD ‑Mutf8 -pi -e 's/<unknown>/@unknown@/g' $i; done
## German
echo "Tagging German proceedings with TreeTagger ...."
for i in $IDIR_DE/*.xml; do echo $i; tree-tagger-german < $i > $ODIR_DE/$(basename $i); done
for i in $ODIR_DE/*.xml; do perl ‑CSAD ‑Mutf8 -pi -e 's/<unknown>/@unknown@/g' $i; done
{'fraction_non_alphanumeric': 0.155, 'fraction_numerical': 0.0015, 'mean_word_length': 4.350267379679145, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 15}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7169090', 'n_tokens_mistral': 804, 'n_tokens_neox': 778, 'n_words': 229}