<commit_before>#! /bin/sh

cat $1 > $2

# Additions
perl -pi -e 's/{\+\+@(\w+) (.+)\+\+}/\\add[$1]{$2}/gm' $2
perl -pi -e 's/{\+\+(.+)\+\+}/\\add{$1}/gm' $2

# Deletions
perl -pi -e 's/{\-\-@(\w+) (.+)\-\-}/\\remove[$1]{$2}/gm' $2
perl -pi -e 's/{\-\-(.+)\-\-}/\\remove{$1}/gm' $2

# Replacements
perl -pi -e 's/{~~@(\w+) (.+)~>(.+)~~}/\\change[$1]{$2}{$3}/gm' $2
perl -pi -e 's/{~~(.+)~>(.+)~~}/\\change{$1}{$2}/gm' $2

# Annotation
perl -pi -e 's/{==(.+)==}{>>@(\w+) (.+)<<}/\\annote[$2]{$1}{$3}/gm' $2
perl -pi -e 's/{==(.+)==}{>>(.+)<<}/\\annote{$1}{$2}/gm' $2<commit_msg>Fix for multi-line comments (add and del only)<commit_after>#! /bin/sh

cat $1 > $2

# Additions
perl -pi -e 's/{\+\+@(\w+) /\\add[$1]{/gm' $2
perl -pi -e 's/{\+\+/\\add{/gm' $2
perl -pi -e 's/\+\+}/}/gm' $2
perl -pi -e 's/{\+\+@(\w+) (.+)\+\+}/\\add[$1]{$2}/gm' $2
perl -pi -e 's/{\+\+(.+)\+\+}/\\add{$1}/gm' $2

# Deletions
perl -pi -e 's/{\-\-@(\w+) /\\remove[$1]{/gm' $2
perl -pi -e 's/{\-\-/\\remove{/gm' $2
perl -pi -e 's/\-\-}/}/gm' $2
perl -pi -e 's/{\-\-@(\w+) (.+)\-\-}/\\remove[$1]{$2}/gm' $2
perl -pi -e 's/{\-\-(.+)\-\-}/\\remove{$1}/gm' $2

# Replacements
perl -pi -e 's/{~~@(\w+) (.+)~>(.+)~~}/\\change[$1]{$2}{$3}/gm' $2
perl -pi -e 's/{~~(.+)~>(.+)~~}/\\change{$1}{$2}/gm' $2

# Annotation
perl -pi -e 's/{==(.+)==}{>>@(\w+) (.+)<<}/\\annote[$2]{$1}{$3}/gm' $2
perl -pi -e 's/{==(.+)==}{>>(.+)<<}/\\annote{$1}{$2}/gm' $2
{'fraction_non_alphanumeric': 0.48903043170559096, 'fraction_numerical': 0.04246284501061571, 'mean_word_length': 2.3747016706443915, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7162416', 'n_tokens_mistral': 836, 'n_tokens_neox': 731, 'n_words': 144}