<commit_before>#include "..\include\model.h"

void Model::init(const char* filename) {
	int lc = line_count(filename);
	std::vector<int> preproc_data = file_preprocess(filename);
	std::vector <std::pair<int, int>> begin_end_markers = file_preprocess_full(preproc_data, lc);

	for (int i = 0; i < begin_end_markers.size(); i++) {
		Mesh mesh;
		load_model(filename, begin_end_markers.back(), mesh);
	}
}

std::vector<std::pair<int, int>> Model::file_preprocess_full(std::vector<int>& preproc_data, int linecount) {
	std::vector <std::pair<int, int>> begin_end_markers;
	for (int i = 0; i < preproc_data.size(); i++) {
		std::pair<int, int> marker;

		if (i == 0)
			marker.first = 3;<commit_msg>fixed model loading with new changes<commit_after>#include "..\include\model.h"

void Model::init(const char* filename) {
	int lc = line_count(filename);
	std::vector<int> preproc_data = file_preprocess(filename);
	std::vector <std::pair<int, int>> begin_end_markers = file_preprocess_full(preproc_data, lc);

	for (int i = 0; i < begin_end_markers.size(); i++) {
		Mesh mesh;
		load_model(filename, begin_end_markers.back(), mesh);
		meshes.push_back(mesh);
	}
}

std::vector<std::pair<int, int>> Model::file_preprocess_full(std::vector<int>& preproc_data, int linecount) {
	std::vector <std::pair<int, int>> begin_end_markers;
	for (int i = 0; i < preproc_data.size(); i++) {
		std::pair<int, int> marker;

		if (i == 0)
			marker.first = 3;
{'fraction_non_alphanumeric': 0.17118997912317327, 'fraction_numerical': 0.0055671537926235215, 'mean_word_length': 5.847619047619047, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1491825', 'n_tokens_mistral': 564, 'n_tokens_neox': 535, 'n_words': 115}