<commit_before>c=0;
for file in `ls datasets/movielens_1m/ratings_dbpedia_train_chunks`;
	
	do

	echo datasets/movielens_1m/ratings_dbpedia_train_chunks/$file;
	#echo $c;
	python -u lib/feature_generator.py -d movielens_1m -e num500_p1_q4_l10_d500.emd --output features/movielens_1m/global_p1_q4/train/training_chunk_$c.svm --training datasets/movielens_1m/ratings_dbpedia_train_chunks/$file &
	c=$((c+1));

	done
<commit_msg>modified parameters and introduced wait for final concatenation<commit_after>c=0;
#sleep 8h;
experiment='movielens_1m';
dataset='ratings_dbpedia_relplusn_val_chunks';
folder='datasets/'$experiment'/'$dataset'/';

for file in `ls $folder`;
	
	do

	echo $folder$file;
	#echo $c;
	python -u lib/feature_generator.py -d $experiment -e num500_p1_q1_l10_d500.emd --output features/$experiment/global_p1_q1/val/val_chunk_$c.svm --training 'datasets/'$experiment'/ratings_dbpedia_train.dat'  --test $folder$file &
	c=$((c+1));

	done

wait

cat features/$experiment/global_p1_q1/val/val_chunk* > features/$experiment/global_p1_q1/val/training_global_p1_q1.svm;
{'fraction_non_alphanumeric': 0.1280148423005566, 'fraction_numerical': 0.038033395176252316, 'mean_word_length': 6.29054054054054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1637643', 'n_tokens_mistral': 463, 'n_tokens_neox': 439, 'n_words': 66}