<commit_before>#include "jayfor.h"

Jayfor *jayforCreate(int argc, char** argv) {
	// not enough args just throw an error
	// if (argc <= 1) {
	// 	printf("error: no input files\n");
	// 	exit(1);
	// }

	Jayfor *jayfor = malloc(sizeof(*jayfor));
	jayfor->scanner = scannerCreate();
	scannerReadFile(jayfor->scanner, "tests/test.j4"); //argv[1]
	jayfor->lexer = lexerCreate(jayfor->scanner->contents);

	return jayfor;
}

void jayforStart(Jayfor *jayfor) {
	do {
		lexerGetNextToken(jayfor->lexer);
	}
	while (jayfor->lexer->running);
}<commit_msg>removed debugging code<commit_after>#include "jayfor.h"

Jayfor *jayforCreate(int argc, char** argv) {
	// not enough args just throw an error
	if (argc <= 1) {
		printf("error: no input files\n");
		exit(1);
	}

	Jayfor *jayfor = malloc(sizeof(*jayfor));
	jayfor->scanner = scannerCreate();
	scannerReadFile(jayfor->scanner, argv[1]);
	jayfor->lexer = lexerCreate(jayfor->scanner->contents);

	return jayfor;
}

void jayforStart(Jayfor *jayfor) {
	do {
		lexerGetNextToken(jayfor->lexer);
	}
	while (jayfor->lexer->running);
}
{'fraction_non_alphanumeric': 0.17116279069767443, 'fraction_numerical': 0.0065116279069767444, 'mean_word_length': 4.663157894736842, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6132427', 'n_tokens_mistral': 455, 'n_tokens_neox': 411, 'n_words': 83}