<commit_before>#!/bin/bash

tmp=/tmp/tex-flatten
cp $1 $tmp

while true; do
    inputs=`grep -o '\\input{.*}' $tmp | cut -f 2 -d '{' | cut -f 1 -d '}'`
    if [ "$inputs" == "" ]; then
        cat $tmp
        exit 0
    fi
   echo $inputs

    # Replace loop
    sedline=""
    for input in $inputs; do
        escinput=`echo $input | sed 's,\/,\\\/,g'`
        sedline="-e \"/input{$escinput}/ {
                  r $input.tex
                  d}\" $sedline"
    done

    /bin/bash -x -c "sed -i $sedline $tmp" &> /dev/null
done

<commit_msg>Do not include lines with comments.<commit_after>#!/bin/bash

tmp=/tmp/tex-flatten
cp $1 $tmp

while true; do
    inputs=`grep -v '%' $tmp | grep -o '\\input{.*}' | cut -f 2 -d '{' | cut -f 1 -d '}'`
    if [ "$inputs" == "" ]; then
        cat $tmp
        exit 0
    fi
   echo $inputs

    # Replace loop
    sedline=""
    for input in $inputs; do
        escinput=`echo $input | sed 's,\/,\\\/,g'`
        sedline="-e \"/input{$escinput}/ {
                  r $input.tex
                  d}\" $sedline"
    done

    /bin/bash -x -c "sed -i $sedline $tmp" &> /dev/null
done


{'fraction_non_alphanumeric': 0.18971631205673758, 'fraction_numerical': 0.0070921985815602835, 'mean_word_length': 1.6134259259259258, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1482907', 'n_tokens_mistral': 441, 'n_tokens_neox': 410, 'n_words': 123}