<commit_before>#include <cctype>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string>
#include <tins/tins.h>

std::string to_lower(std::string str)
{
	for(auto& ii:str)
		ii=tolower(ii);

	return str;
}

void save_to_file(const std::string& data,const std::string& file)
{
	std::ofstream ostr(file);
	ostr<<data;
	ostr.close();
	std::cout<<"\tSaved:\t\t"<<file<<std::endl;
}

void save_payload(const Tins::TCPStream::payload_type& payload,const std::string& name)
{
	save_to_file(std::string((char*)payload.data(),payload.size()),name+".raw");
}

void process_html(const Tins::RawPDU::payload_type& payload,const std::string& name)
{
	std::string data((char*)payload.data(),payload.size());
	size_t count=0;
	size_t ptr=0;

	while(true)
	{
		bool found=false;

		if(data.substr(ptr,15)=="HTTP/1.0 200 OK"||data.substr(ptr,15)=="HTTP/1.1 200 OK")
		{
			size_t cl_ptr=to_lower(data).find("content-length:",ptr);
			size_t header_end=data.find("\r\n\r\n",ptr);

			if(cl_ptr!=std::string::npos&&header_end!=std::string::npos)
			{
				std::istringstream istr(data.substr(cl_ptr+15,data.size()));
				size_t html_size=0;

				if((istr>>html_size)&&html_size>0&&html_size<data.size())
				{
					save_to_file(data.substr(header_end+4,html_size),name+"_"+std::to_string(count)+".html");
					ptr+=header_end+4+html_size;
					++count;
					found=true;
				}
			}
		}

		if(!found||ptr>=data.size())
			break;
	}
}

void process_contiguous_nops(const Tins::RawPDU::payload_type& payload,const std::string& name)
{
	size_t max_count=0;
	size_t count=0;

	for(size_t ii=0;ii<payload.size();++ii)
	{
		if(payload[ii]!=0x90||ii+1>=payload.size())
		{
			if(count>max_count)
				max_count=count;

			count=0;
			continue;
		}

		++count;
	}

	std::cout<<"\tNOP Contiguous:\t"<<count<<std::endl;
}

void process_nops(const Tins::RawPDU::payload_type& payload,const std::string& name)
{
	size_t count=0;

	for(size_t ii=0;ii<payload.size();++ii)
		if(payload[ii]==0x90)
			++count;<commit_msg>Yea...last commit to the actual source should probably actually git committed and pushed...<commit_after>#include <cctype>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <string>
#include <tins/tins.h>

std::string to_lower(std::string str)
{
	for(auto& ii:str)
		ii=tolower(ii);

	return str;
}

void save_to_file(const std::string& data,const std::string& file)
{
	std::ofstream ostr(file);
	ostr<<data;
	ostr.close();
	std::cout<<"\tSaved:\t\t"<<file<<std::endl;
}

void save_payload(const Tins::TCPStream::payload_type& payload,const std::string& name)
{
	save_to_file(std::string((char*)payload.data(),payload.size()),name+".raw");
}

void process_html(const Tins::RawPDU::payload_type& payload,const std::string& name)
{
	std::string data((char*)payload.data(),payload.size());
	size_t count=0;
	size_t ptr=0;

	while(true)
	{
		bool found=false;

		if(data.substr(ptr,15)=="HTTP/1.0 200 OK"||data.substr(ptr,15)=="HTTP/1.1 200 OK")
		{
			size_t cl_ptr=to_lower(data).find("content-length:",ptr);
			size_t header_end=data.find("\r\n\r\n",ptr);

			if(cl_ptr!=std::string::npos&&header_end!=std::string::npos)
			{
				std::istringstream istr(data.substr(cl_ptr+15,data.size()));
				size_t html_size=0;

				if((istr>>html_size)&&html_size>0&&html_size<data.size())
				{
					save_to_file(data.substr(header_end+4,html_size),name+"_"+std::to_string(count)+".html");
					ptr+=header_end+4+html_size;
					++count;
					found=true;
				}
			}
		}

		if(!found||ptr>=data.size())
			break;
	}
}

void process_contiguous_nops(const Tins::RawPDU::payload_type& payload,const std::string& name)
{
	size_t max_count=0;
	size_t count=0;

	for(size_t ii=0;ii<payload.size();++ii)
	{
		if(payload[ii]!=0x90||ii+1>=payload.size())
		{
			if(count>max_count)
				max_count=count;

			count=0;
			continue;
		}

		++count;
	}

	std::cout<<"\tNOP Contiguous:\t"<<max_count<<std::endl;
}

void process_nops(const Tins::RawPDU::payload_type& payload,const std::string& name)
{
	size_t count=0;

	for(size_t ii=0;ii<payload.size();++ii)
		if(payload[ii]==0x90)
			++count;
{'fraction_non_alphanumeric': 0.19757869249394674, 'fraction_numerical': 0.01694915254237288, 'mean_word_length': 5.695299837925446, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7102574', 'n_tokens_mistral': 1861, 'n_tokens_neox': 1687, 'n_words': 227}