<commit_before>#include "tests.h"
#include "Allocator.h"
#include "stream.h"
#include "Parser.h"
#include <sstream>

void test_ostream()
{
	Allocator a;
	pcell_t p = new (a) Cell<value_t,pcell_t>{
					0,
					new (a) Cell<pcell_t,value_t>{
						new (a) Cell<value_t,value_t>{3,3},
						2 } };

	std::ostringstream os;
	KConOStream( os ) << "flat = " << p;
	KConOStream( os ) << "deep = " << deep << p;
	test( os.str() == "flat = [0 [3 3] 2]deep = [0 [[3 3] 2]]", "Incorrect printing found '" + os.str() + "'" );
}

void test_parser()
{
	Allocator a;
	std::istringstream iss("[0 [1 [2 3] 4] 5 6]");
	//KConOStream( std::cout ) << Parser( a ).parse( iss );
}

void test_io( std::string in, KConOStream::Manip m, std::string out="" )
{
	if( out.empty() )
		out = in;
		
	Allocator a;
	std::istringstream iss( in );
	std::ostringstream oss;
	KConOStream( oss ) << m << Parser( a ).parse( iss );
	
	test( oss.str() == out, "IO failed for: " + in + "\nExpected: " + out + "\nFound: " + oss.str() );
}

void test_stream()
{
	test_io( "[0 0]", flat, "3" );<commit_msg>Extra io tests<commit_after>#include "tests.h"
#include "Allocator.h"
#include "stream.h"
#include "Parser.h"
#include <sstream>

void test_ostream()
{
	Allocator a;
	pcell_t p = new (a) Cell<value_t,pcell_t>{
					0,
					new (a) Cell<pcell_t,value_t>{
						new (a) Cell<value_t,value_t>{3,3},
						2 } };

	std::ostringstream os;
	KConOStream( os ) << "flat = " << p;
	KConOStream( os ) << "deep = " << deep << p;
	test( os.str() == "flat = [0 [3 3] 2]deep = [0 [[3 3] 2]]", "Incorrect printing found '" + os.str() + "'" );
}

void test_io( std::string in, KConOStream::Manip m, std::string out="" )
{
	if( out.empty() )
		out = in;
		
	Allocator a;
	std::istringstream iss( in );
	std::ostringstream oss;
	KConOStream( oss ) << m << Parser( a ).parse( iss );
	
	test( oss.str() == out, "IO failed for: " + in + "\nExpected: " + out + "\nFound:    " + oss.str() );
}

void test_stream()
{
	test_io( "[0 [1 2]]", flat, "[0 1 2]" );
	test_io( "[0 1 2]", deep, "[0 [1 2]]" );
	test_io( "[0 [1 [2 3] 4] 5 6]", flat );
	test_io( "[0 [1 [2 3] 4] 5 6]", deep, "[0 [[1 [[2 3] 4]] [5 6]]]" );
{'fraction_non_alphanumeric': 0.22511627906976744, 'fraction_numerical': 0.031162790697674417, 'mean_word_length': 3.5, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4589127', 'n_tokens_mistral': 1001, 'n_tokens_neox': 913, 'n_words': 250}