<commit_before>#!/bin/bash

set -e
set -o pipefail

trap 'echo ERROR; read a' ERR

if [ "$1"x = ""x ]; then
	echo "Start up"
	cd "$(dirname "$0")"
	bash "$(pwd)"/"$(basename "$0")" "$(pwd)"
	exit $?
fi

cd "$1"
cwd="$(pwd)"

echo "... $cwd"

cp /dev/null 0_files.txt
for dir in *; do
	if [ -d "$dir" ]; then bash "$0" "$dir" || exit $?; continue; fi
	dir="$(basename "$dir")"
	if [ "$dir" = "0_files.txt" ]; then continue; fi
	echo "$(basename "$dir")" >> 0_files.txt
done
<commit_msg>add automatic .gitignore file creation as the generated 0_files.txt files don't belong in repo<commit_after>#!/bin/bash

set -e
set -o pipefail

trap 'echo ERROR; read a' ERR

if [ "$1"x = ""x ]; then
	echo "Start up"
	cd "$(dirname "$0")"
	bash "$(pwd)"/"$(basename "$0")" "$(pwd)"
	exit $?
fi

cd "$1"
cwd="$(pwd)"

if [ ! -f .gitignore ]; then touch .gitignore; fi
grep "/0_files.txt" .gitignore > /dev/null 2>&1 || echo "/0_files.txt" >> .gitignore  

echo "... $cwd"

cp /dev/null 0_files.txt
for dir in *; do
	if [ -d "$dir" ]; then bash "$0" "$dir" || exit $?; continue; fi
	dir="$(basename "$dir")"
	if [ "$dir" = "0_files.txt" ]; then continue; fi
	echo "$(basename "$dir")" >> 0_files.txt
done

{'fraction_non_alphanumeric': 0.2304457527333894, 'fraction_numerical': 0.017661900756938603, 'mean_word_length': 3.5593869731800765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5001974', 'n_tokens_mistral': 513, 'n_tokens_neox': 485, 'n_words': 154}