<commit_before>#include <climits>
#include <stdio.h>
#define LTC_SHA1
#define SHA1
#include <tomcrypt.h>
#include <algorithm>
#include "util.hpp"

std::vector<std::string> split(const std::string &str, const std::string &split_by, unsigned int max_count = UINT_MAX)
{
	typedef std::vector<std::string> string_list;
	string_list res;
	
	std::string::const_iterator it = str.begin();
	while (it != str.end())
	{
		res.push_back(std::string());
		std::string &part = res.back();
		
		std::string::const_iterator next_it = std::search(it, str.end(), split_by.begin(), split_by.end());
		std::copy(it, next_it, std::back_inserter(part));
		if (next_it != str.end()) // if we found an instance of split_by, we want to move forward to skip that as well.
			it = next_it + split_by.length();
		else // otherwise, just let the loop terminate.
			it = next_it;
	}
	return res;
}

std::vector<unsigned char> sha1(const std::string &str)
{
	std::vector<unsigned char> res(20, 0);<commit_msg>Make split() actually limit itself to the number of columns it's supposed to.<commit_after>#include <climits>
#include <stdio.h>
#define LTC_SHA1
#define SHA1
#include <tomcrypt.h>
#include <algorithm>
#include "util.hpp"

std::vector<std::string> split(const std::string &str, const std::string &split_by, unsigned int max_count = UINT_MAX)
{
	typedef std::vector<std::string> string_list;
	string_list res;
	
	std::string::const_iterator it = str.begin(), next_it;
	while (it != str.end())
	{
		res.push_back(std::string());
		std::string &part = res.back();
		
		if (--max_count) // only search for the next split token if we're supposed to be finding more.
 			next_it = std::search(it, str.end(), split_by.begin(), split_by.end());
		else
			next_it = str.end();
			
		std::copy(it, next_it, std::back_inserter(part));
		if (next_it != str.end()) // if we found an instance of split_by, we want to move forward to skip that as well.
			it = next_it + split_by.length();
		else // otherwise, just let the loop terminate.
			it = next_it;
	}
	return res;
}

std::vector<unsigned char> sha1(const std::string &str)
{
	std::vector<unsigned char> res(20, 0);
{'fraction_non_alphanumeric': 0.15535797847449695, 'fraction_numerical': 0.005615348619560131, 'mean_word_length': 4.778378378378378, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7400051', 'n_tokens_mistral': 795, 'n_tokens_neox': 738, 'n_words': 214}