<commit_before>#!/bin/bash


for f in grep -l '## References$' *.markdown *.md 
  # *md *.markdown 
  do 
    sed -n '/---/,/---/p' $f > yaml.md
    sed  '1,/^$/d' $f > body.md
    ~/.cabal/bin/./pandoc -f markdown body.md -t html -o body.html
    ~/.cabal/bin/./pandoc body.html -t markdown_github -o out.md
    cat yaml.md out.md > $f
    rm yaml.md body.md out.md body.html 
done

<commit_msg>fix citations<commit_after>#!/bin/bash


for f in `grep -l '##* References$' *.markdown *.md`
  # *md *.markdown 
  do 
    sed -i 's/##* References/\n## References' $f 
    sed -n '/---/,/---/p' $f > yaml.md
    sed  '1,/^$/d' $f > body.md
    ~/.cabal/bin/./pandoc -f markdown body.md -t html --mathjax -o body.html
    ~/.cabal/bin/./pandoc body.html -t markdown_github --mathjax -o out.md
    cat yaml.md out.md > $f
    rm yaml.md body.md out.md body.html 
done


{'fraction_non_alphanumeric': 0.2048611111111111, 'fraction_numerical': 0.0023148148148148147, 'mean_word_length': 1.9827586206896552, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4500136', 'n_tokens_mistral': 398, 'n_tokens_neox': 360, 'n_words': 99}