<commit_before>for file in "$TMPDIR"$uuid"/xtarget."*
do

		"$PYTHON27_BIN" $scriptpath"bin/nerv3.py" $file $file"_nouns.txt" "$uuid"
		echo "ran nerv3 on $file" | tee --append $sfb_log
		cat "$TMPDIR$uuid"/Places >> "$TMPDIR"$batch_uuid"/"$sku"."$safe_product_name"_Places"
		cat "$TMPDIR$uuid"/People >>  "$TMPDIR"$batch_uuid"/"$sku"."$safe_product_name"_People"
		cat "$TMPDIR$uuid"/Other >>  "$TMPDIR"$batch_uuid"/"$sku"."$safe_product_name"_Other"

		echo "python_bin for running PKsum is" $PYTHON_BIN "and PYTHON_BIN actually is"
		"$PYTHON_BIN" --version

		"$PYTHON_BIN" bin/PKsum.py -l "$summary_length" -o $file"_summary.txt" $file
		sed -i 's/ \+ / /g' $file"_summary.txt"
		cp $file"_summary.txt" $file"_pp_summary.txt"
		echo "ran summarizer on $file" | tee --append $sfb_log
		awk 'length>=50' $file"_pp_summary.txt" >  "$TMPDIR"$uuid/awk.tmp && mv  "$TMPDIR"$uuid/awk.tmp $file"_pp_summary.txt"
		awk 'length<=4000' $file"_pp_summary.txt" >  "$TMPDIR"$uuid/awk.tmp && mv  "$TMPDIR"$uuid/awk.tmp $file"_pp_summary.txt"
		#echo "---end of summary section of 140K bytes---" >> $file"_pp_summary.txt"
		#echo "---end of summary section of 140K bytes---" >> $file"_summary.txt"
		cat $file"_pp_summary.txt" | awk '!x[$0]++' >>  "$TMPDIR"$uuid/pp_summary.txt
		cat $file"_summary.txt" | awk '!x[$0]++' >>  "$TMPDIR"$uuid/summary.txt
done<commit_msg>removes extraneous NER file builds<commit_after>for file in "$TMPDIR"$uuid"/xtarget."*
do

		"$PYTHON27_BIN" $scriptpath"bin/nerv3.py" $file $file"_nouns.txt" "$uuid"
		echo "ran nerv3 on $file" | tee --append $sfb_log
		cat "$TMPDIR$uuid"/Places >> "$TMPDIR"$uuid"/"$sku"."$safe_product_name"_Places"
		cat "$TMPDIR$uuid"/People >>  "$TMPDIR"$uuid"/"$sku"."$safe_product_name"_People"
		cat "$TMPDIR$uuid"/Other >>  "$TMPDIR"$uuid"/"$sku"."$safe_product_name"_Other"

		echo "python_bin for running PKsum is" $PYTHON_BIN "and PYTHON_BIN actually is"
		"$PYTHON_BIN" --version

		"$PYTHON_BIN" bin/PKsum.py -l "$summary_length" -o $file"_summary.txt" $file
		sed -i 's/ \+ / /g' $file"_summary.txt"
		cp $file"_summary.txt" $file"_pp_summary.txt"
		echo "ran summarizer on $file" | tee --append $sfb_log
		awk 'length>=50' $file"_pp_summary.txt" >  "$TMPDIR"$uuid/awk.tmp && mv  "$TMPDIR"$uuid/awk.tmp $file"_pp_summary.txt"
		awk 'length<=4000' $file"_pp_summary.txt" >  "$TMPDIR"$uuid/awk.tmp && mv  "$TMPDIR"$uuid/awk.tmp $file"_pp_summary.txt"
		#echo "---end of summary section of 140K bytes---" >> $file"_pp_summary.txt"
		#echo "---end of summary section of 140K bytes---" >> $file"_summary.txt"
		cat $file"_pp_summary.txt" | awk '!x[$0]++' >>  "$TMPDIR"$uuid/pp_summary.txt
		cat $file"_summary.txt" | awk '!x[$0]++' >>  "$TMPDIR"$uuid/summary.txt
done
{'fraction_non_alphanumeric': 0.20301581463773447, 'fraction_numerical': 0.013240161824200073, 'mean_word_length': 4.79957356076759, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '358231', 'n_tokens_mistral': 1208, 'n_tokens_neox': 1135, 'n_words': 215}