<commit_before>rm -r output
mkdir output
for mdbfile in $(ls input); do
	dbname=$(echo $mdbfile | cut -f 1 -d '.')
	mkdir output/$dbname
	mkdir output/$dbname/schema
	mkdir output/$dbname/inserts

	echo "Generating schema for $mdbfile"
	mdb-schema input/$mdbfile sqlite > output/$dbname/schema/$dbname.sql
	for table in $(mdb-tables input/$mdbfile); do
		echo "Generating inserts for $dbname::$table"
		mdb-export -I sqlite input/$mdbfile $table > output/$dbname/inserts/$table.sql
	done
done

<commit_msg>Update script to properly handle converting datetimes to a valid sqlite format.<commit_after>rm -r output
mkdir output
for mdbfile in $(ls input); do
	dbname=$(echo $mdbfile | cut -f 1 -d '.')
	mkdir output/$dbname
	mkdir output/$dbname/schema
	mkdir output/$dbname/inserts

	echo "Generating schema for $mdbfile"
	mdb-schema input/$mdbfile sqlite > output/$dbname/schema/$dbname.sql
	for table in $(mdb-tables input/$mdbfile); do
		echo "Generating inserts for $dbname::$table"
		mdb-export -D "%Y-%m-%d %H:%M:%S" -I sqlite input/$mdbfile $table > output/$dbname/inserts/$table.sql
	done
done


{'fraction_non_alphanumeric': 0.1307901907356948, 'fraction_numerical': 0.0018165304268846503, 'mean_word_length': 4.375609756097561, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '335003', 'n_tokens_mistral': 401, 'n_tokens_neox': 378, 'n_words': 116}