<commit_before>#include "Page.h"

Page::Page(string html)
{
	cout << "bulding page..." << endl;
	makeText(html);
}

Page::~Page()
{
	cout << "killing Page..." <<endl;
}


void Page::makeText(string html)
{
	bool foundTitle = false;
	bool inBody = false;
	string temp = "";
	for (int i = 0; i < html.length() ; i++)
	{
		if(!foundTitle)
		{
			if(html[i] == '<')
			{
				//title>
				i++;
				string titleTag = html.substr(i, 5); //legth of title
				if(titleTag == "title") //check if the tag is title
				{
					while(html[i] != '>')
						i++;
					i++;
					while(html[i] != '<')
					{
						temp += html[i];
						i++;
					}
					foundTitle = true;
					this->title = temp;
				}
			}
			temp = "" ; //clear temp BUFFER
		}
		else
		{
			if(html[i] == '<')
			{
				//body>
				i++;
				string bodyTag = html.substr(i, 4); //legth of "body"
				if(bodyTag == "body") //check if the tag is body tag
				{
					while(html[i] != '>')
						i++;
					i++;
					inBody = true;
				}
			}
		}
		
		if(inBody)
		{
			if(html[i] == '>')
			{
				i++;
				while(html[i] != '<')
				{
					temp += html[i];
					i++;
				}
			}
		}
	}<commit_msg>removed Tabs -> cleaning text some more<commit_after>#include "Page.h"

Page::Page(string html)
{
	cout << "bulding page..." << endl;
	makeText(html);
}

Page::~Page()
{
	cout << "killing Page..." <<endl;
}


void Page::makeText(string html)
{
	bool foundTitle = false;
	bool inBody = false;
	string temp = "";
	for (int i = 0; i < html.length() ; i++)
	{
		if(!foundTitle)
		{
			if(html[i] == '<')
			{
				//title>
				i++;
				string titleTag = html.substr(i, 5); //legth of title
				if(titleTag == "title") //check if the tag is title
				{
					while(html[i] != '>')
						i++;
					i++;
					while(html[i] != '<')
					{
						temp += html[i];
						i++;
					}
					foundTitle = true;
					this->title = temp;
				}
			}
			temp = "" ; //clear temp BUFFER
		}
		else
		{
			if(html[i] == '<')
			{
				//body>
				i++;
				string bodyTag = html.substr(i, 4); //legth of "body"
				if(bodyTag == "body") //check if the tag is body tag
				{
					while(html[i] != '>')
						i++;
					i++;
					inBody = true;
				}
			}
		}
		
		if(inBody)
		{
			if(html[i] == '>')
			{
				i++;
				while(html[i] != '<')
				{
						temp += html[i];
					i++;
				}
			}
		}
	}
{'fraction_non_alphanumeric': 0.21530479896238652, 'fraction_numerical': 0.0025940337224383916, 'mean_word_length': 1.8816936488169365, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6280141', 'n_tokens_mistral': 1223, 'n_tokens_neox': 919, 'n_words': 201}