<commit_before>#include <stdio.h>
#include <string.h>
#include "JSONParser.h"
#include "Filesystem.h"
#include "FileStream.h"

using namespace crown;

int main(int argc, char** argv)
{
	if (argc != 2)
	{
		printf("Configuration root path must be provided. Aborting!");
		return -1;
	}

	Filesystem conf_root(argv[1]);

	if (!conf_root.exists("json.json"))
	{
		printf("Configuration file does not exists. Aborting!\n");
		return -1;
	}
	
	char dst[10][256];

	FileStream* stream = (FileStream*)conf_root.open("json.json", SOM_READ);

 	JSONParser* parser = new JSONParser(stream);

	json_error error;
	JSONToken* tokens;

	parser->init();
	error = parser->parse();
	tokens = parser->get_tokens();

	for (int i = 0; i < parser->get_tokens_number(); i++)
	{
		printf("token[%d]\n", i);
		printf("type: %d\n", tokens[i].m_type);
		printf("size: %d\n", tokens[i].m_size);
		printf("start: %d\n",tokens[i].m_start);
		printf("end: %d\n",tokens[i].m_end);
		printf("parent token: %d\n",tokens[i].m_parent);
		printf("\n");
	}

	parser->shutdown();

	printf("return: %d\n", error);

	conf_root.close(stream);

	return 0;
}<commit_msg>json samples adapted to recent changes<commit_after>#include <stdio.h>
#include <string.h>
#include "JSONParser.h"
#include "Filesystem.h"
#include "FileStream.h"

using namespace crown;

int main(int argc, char** argv)
{
	if (argc != 2)
	{
		printf("Configuration root path must be provided. Aborting!");
		return -1;
	}

	Filesystem conf_root(argv[1]);

	if (!conf_root.exists("json.json"))
	{
		printf("Configuration file does not exists. Aborting!\n");
		return -1;
	}
	
	char dst[10][256];

	FileStream* stream = (FileStream*)conf_root.open("json.json", SOM_READ);

 	JSONParser* parser = new JSONParser(stream);

	JSONError error;
	JSONToken* tokens;

	error = parser->parse();
	tokens = parser->get_tokens();

	for (int i = 0; i < parser->get_tokens_number(); i++)
	{
		printf("token[%d]\n", i);
		printf("type: %d\n", tokens[i].m_type);
		printf("size: %d\n", tokens[i].m_size);
		printf("start: %d\n",tokens[i].m_start);
		printf("end: %d\n",tokens[i].m_end);
		printf("parent token: %d\n",tokens[i].m_parent);
		printf("\n");
	}

	printf("return: %d\n", error);

	conf_root.close(stream);

	return 0;
}
{'fraction_non_alphanumeric': 0.19562304600267977, 'fraction_numerical': 0.00982581509602501, 'mean_word_length': 4.670886075949367, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5792837', 'n_tokens_mistral': 903, 'n_tokens_neox': 836, 'n_words': 185}