<commit_before>#!/bin/bash

debuginfodir="../../debuginfo"

separator="-------SEPARATOR--------"

for dump in coredumps/*.dmp; do

	rev=`Win32MinidumpAnalyzer.exe "$dump" 0 | grep -E -o '_r[0-9]+' | sed 's/_//'`
	if [ -z "$rev" ]; then
		rev=`Win32MinidumpAnalyzer.exe "$dump" 0 | grep -E -o '_beta[0-9]+' | sed 's/_//'`
	fi
	if [ -z "$rev" ]; then
		echo Coredump $dump rev `Win32MinidumpAnalyzer.exe "$dump" 0` - cannot determine revnumber > "$dump.txt"
		echo $separator >> "$dump.txt"
		continue
	fi
	echo Coredump $dump rev $rev > "$dump.txt"
	echo $separator >> "$dump.txt"
	Win32MinidumpAnalyzer.exe "$dump" 1 >> "$dump.txt"
	echo $separator >> "$dump.txt"
	Win32MinidumpAnalyzer.exe "$dump" 2 >> "$dump.txt"
	echo $separator >> "$dump.txt"
	
	if [ \! -e "$debuginfodir/$rev" ] ; then
		echo No debuginfo for rev $rev >> "$dump.txt"
		revnumActual=`echo $rev | grep -E -o '[0-9]+'`
		revnumMinDiff=100000000
		for f in `ls $debuginfodir` ; do
			revnumCheck=`echo $f | grep -E -o '[0-9]+'`
			revnumDiff=`expr $revnumCheck - $revnumActual | sed 's/-//'`
			if [ $revnumDiff -lt $revnumMinDiff ] ; then
				revnumMinDiff=$revnumDiff
				rev=$f
			fi
		done
		if [ \! -e "$debuginfodir/$rev" ] ; then
			echo Cannot find any debuginfo for rev $rev >> "$dump.txt"
			echo $separator >> "$dump.txt"
			continue
		fi
		echo Found closest matching debuginfo - rev $rev >> "$dump.txt"
		echo $separator >> "$dump.txt"
	fi
	
	cdb -y "$debuginfodir/$rev" -i "$debuginfodir/$rev" -z "$dump" \
		-srcpath "$debuginfodir/$rev/sources" -lines \
		-c ".kframes 100; .ecxr; kp; q" > except.txt
	fileline=`grep -o -m 1 '[^[]*@ [0-9]*' except.txt`
	rm except.txt

	if [ -z "$fileline" ]; then
		echo Cannot determine exception location in source files >> "$dump.txt"
		echo $separator >> "$dump.txt"
		cdb -y "$debuginfodir/$rev" -i "$debuginfodir/$rev" -z "$dump" \
			-srcpath "$debuginfodir/$rev/sources" -lines \
			-c ".kframes 100; .ecxr; kp; ~*kp; q" >> "$dump.txt",
		echo $separator >> "$dump.txt"
	else
		file=`echo $fileline | grep -o '[^@]*' | sed 's/ //'`
		line=`echo $fileline | grep -o '@ [0-9]*' | sed 's/@ //'`
		echo Exception at $file: $line >> "$dump.txt"
		echo $separator >> "$dump.txt"
		if [ $line -gt 10 ]; then
			line=`expr $line - 10`
		else
			line=0
		fi
		cdb -y "$debuginfodir/$rev" -i "$debuginfodir/$rev" -z "$dump" \
			-srcpath "$debuginfodir/$rev/sources" -lines \
			-c ".kframes 100; lsp -a 14; lsf $file; ls $line; .ecxr; kp; ~*kp; q" >> "$dump.txt"
		echo $separator >> "$dump.txt"
	fi
done<commit_msg>Updated coredump generator to generate more parser-friendly files<commit_after>#!/bin/bash

debuginfodir="../../debuginfo"

separator="-------SEPARATOR--------"

for dump in coredumps/*.dmp; do

	rev=`Win32MinidumpAnalyzer.exe "$dump" 0 | grep -E -o '_r[0-9]+' | sed 's/_//'`
	if [ -z "$rev" ]; then
		rev=`Win32MinidumpAnalyzer.exe "$dump" 0 | grep -E -o '_beta[0-9]+' | sed 's/_//'`
	fi
	if [ -z "$rev" ]; then
		echo Coredump $dump rev `Win32MinidumpAnalyzer.exe "$dump" 0` - cannot determine revnumber > "$dump.txt"
		echo $separator >> "$dump.txt"
		continue
	fi
	echo $dump > "$dump.txt"
	echo $rev >> "$dump.txt"
	echo $separator >> "$dump.txt"
	Win32MinidumpAnalyzer.exe "$dump" 1 >> "$dump.txt"
	echo $separator >> "$dump.txt"
	Win32MinidumpAnalyzer.exe "$dump" 2 >> "$dump.txt"
	echo $separator >> "$dump.txt"
	
	if [ \! -e "$debuginfodir/$rev" ] ; then
		echo No debuginfo for rev $rev >> "$dump.txt"
		revnumActual=`echo $rev | grep -E -o '[0-9]+'`
		revnumMinDiff=100000000
		for f in `ls $debuginfodir` ; do
			revnumCheck=`echo $f | grep -E -o '[0-9]+'`
			revnumDiff=`expr $revnumCheck - $revnumActual | sed 's/-//'`
			if [ $revnumDiff -lt $revnumMinDiff ] ; then
				revnumMinDiff=$revnumDiff
				rev=$f
			fi
		done
		if [ \! -e "$debuginfodir/$rev" ] ; then
			echo Cannot find any debuginfo for rev $rev >> "$dump.txt"
			echo $separator >> "$dump.txt"
			continue
		fi
		echo Found closest matching debuginfo - rev $rev >> "$dump.txt"
		echo $separator >> "$dump.txt"
	fi
	
	cdb -y "$debuginfodir/$rev" -i "$debuginfodir/$rev" -z "$dump" \
		-srcpath "$debuginfodir/$rev/sources" -lines \
		-c ".kframes 100; .ecxr; kp; q" > except.txt
	fileline=`grep -o -m 1 '[^[]*@ [0-9]*' except.txt`
	rm except.txt

	if [ -z "$fileline" ]; then
		echo Cannot determine exception location in source files >> "$dump.txt"
		echo $separator >> "$dump.txt"
		cdb -y "$debuginfodir/$rev" -i "$debuginfodir/$rev" -z "$dump" \
			-srcpath "$debuginfodir/$rev/sources" -lines \
			-c ".kframes 100; .echo -----ECXR-----; .ecxr; .echo -----CRASHED_THREAD_STACK-----; kp; .echo -----THREADS_STACK-----; ~*kp; q" >> "$dump.txt"
		echo $separator >> "$dump.txt"
	else
		file=`echo $fileline | grep -o '.*@' | sed 's/@//' | sed 's/ //'`
		line=`echo $fileline | grep -o '@ [0-9]*' | sed 's/@ //'`
		echo Exception at $file: $line >> "$dump.txt"
		echo $separator >> "$dump.txt"
		if [ $line -gt 10 ]; then
			line=`expr $line - 10`
		else
			line=0
		fi
		cdb -y "$debuginfodir/$rev" -i "$debuginfodir/$rev" -z "$dump" \
			-srcpath "$debuginfodir/$rev/sources" -lines \
			-c ".kframes 100; lsp -a 14; .echo -----SOURCE-----; lsf $file; ls $line; .echo -----ECXR-----; .ecxr; .echo -----CRASHED_THREAD_STACK-----; kp; .echo -----THREADS_STACK-----; ~*kp; q" >> "$dump.txt"
		echo $separator >> "$dump.txt"
	fi
done
{'fraction_non_alphanumeric': 0.213844714686623, 'fraction_numerical': 0.01983161833489242, 'mean_word_length': 2.971768202080238, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2108449', 'n_tokens_mistral': 2370, 'n_tokens_neox': 2082, 'n_words': 576}