<commit_before>//@	{"targets":[{"name":"itemtree.hpp","type":"include"}]}

#ifndef AJSON_ITEMTREE_HPP
#define AJSON_ITEMTREE_HPP

#include "numberformat.hpp"
#include "type.hpp"

#include <map>
#include <vector>
#include <stack>
#include <cassert>
#include <cstddef>

namespace Temple
	{
	struct BasicStorage
		{
		template<class T>
		using ArrayType=std::vector<T>;
		using StringType=std::string;
		using BufferType=std::string;
		};

	template<class StorageModel=BasicStorage>
	class ItemTree
		{
		public:
			template<class T>
			using ArrayType=typename StorageModel::template ArrayType<T>;

			using StringType=typename StorageModel::StringType;

			using BufferType=typename StorageModel::BufferType;

			static constexpr typename StringType::value_type pathsep() noexcept
				{return '\001';}

			template<class Source,class ProgressMonitor>
			ItemTree(Source&& src,ProgressMonitor&& monitor)
				{load(src,monitor);}

			template<class Source,class ProgressMonitor>
			ItemTree& load(Source&& src,ProgressMonitor& monitor)
				{return load(src,monitor);}

			template<class Source,class ProgressMonitor>
			ItemTree& load(Source& src,ProgressMonitor& monitor);

			template<class ItemProcessor,class ExceptionHandler>
			void itemsProcess(ItemProcessor&& proc,ExceptionHandler&& eh);

			template<class Sink,class ExceptionHandler>
			void store(Sink& sink,ExceptionHandler& eh)
				{
				StringType path_prev;
				itemsProcess([this,&sink,&path_prev,&eh](const auto& key,auto tag,const auto& value)
					{
					auto path_end=this->rfind(key,pathsep());
					auto level=this->count(key,pathsep());
					RecordWrite<decltype(tag)::id>::doIt(sink,level,path_end,value,eh);
				//	RecordWrite<decltype(tag)::id>::write(path_end,value,sink);
				//	if(path_end!=nullptr)
				//		{
				//		fprintf(sink,"[%s] %zu\n",key.c_str(),level);
				//		}
					},eh);
				}

			template<Type t>
			auto& dataGet() noexcept
				{return dataGet<t>(m_data);}

			template<Type t>
			const auto& dataGet() const noexcept
				{return dataGet<t>(m_data);}

		private:
			template<class T>
			static constexpr bool equals_or(T x)
				{return 0;}

			template<class T,class ... U>
			static constexpr bool equals_or(T x,T y,U ... values)
				{return x==y || equals_or(x,values...);}

			template<class Cstr,class Stream,class... to_escape>
			static void write(Cstr src,Stream& stream,to_escape ... esc)
				{
				while(true)
					{
					auto ch_in=*src;
					if(ch_in=='\0')
						{return;}
					if(equals_or(ch_in,esc...))
						{putc('\\',stream);}
					putc(ch_in,stream);
					++src;
					}
				}

			template<Type t,class dummy=void>
			struct RecordWrite
				{
				template<class Sink,class KeyCstr,class Value,class ExceptionHandler>
				static void doIt(Sink& sink,size_t level,KeyCstr key,const Value& value,ExceptionHandler eh)
					{
					if(key!=nullptr)
						{
						assert(level!=0);
						for(size_t k=0;k<level-1;++k)
							{putc('\t',sink);}
						putc('"',sink);
						write(key,sink,'"');
						fprintf(sink,"\":\n");
						}
					}
				};

			using Key=StringType;<commit_msg>Speciallizing RecordWrite<commit_after>//@	{"targets":[{"name":"itemtree.hpp","type":"include"}]}

#ifndef AJSON_ITEMTREE_HPP
#define AJSON_ITEMTREE_HPP

#include "numberformat.hpp"
#include "type.hpp"

#include <map>
#include <vector>
#include <stack>
#include <cassert>
#include <cstddef>

namespace Temple
	{
	struct BasicStorage
		{
		template<class T>
		using ArrayType=std::vector<T>;
		using StringType=std::string;
		using BufferType=std::string;
		};

	template<class StorageModel=BasicStorage>
	class ItemTree
		{
		public:
			template<class T>
			using ArrayType=typename StorageModel::template ArrayType<T>;

			using StringType=typename StorageModel::StringType;

			using BufferType=typename StorageModel::BufferType;

			static constexpr typename StringType::value_type pathsep() noexcept
				{return '\001';}

			template<class Source,class ProgressMonitor>
			ItemTree(Source&& src,ProgressMonitor&& monitor)
				{load(src,monitor);}

			template<class Source,class ProgressMonitor>
			ItemTree& load(Source&& src,ProgressMonitor& monitor)
				{return load(src,monitor);}

			template<class Source,class ProgressMonitor>
			ItemTree& load(Source& src,ProgressMonitor& monitor);

			template<class ItemProcessor,class ExceptionHandler>
			void itemsProcess(ItemProcessor&& proc,ExceptionHandler&& eh);

			template<class Sink,class ExceptionHandler>
			void store(Sink& sink,ExceptionHandler& eh)
				{
				StringType path_prev;
				itemsProcess([this,&sink,&path_prev,&eh](const auto& key,auto tag,const auto& value)
					{
					auto path_end=this->rfind(key,pathsep());
					auto level=this->count(key,pathsep());
					RecordWrite<decltype(tag)::id>::doIt(sink,level,path_end,value,eh);
				//	RecordWrite<decltype(tag)::id>::write(path_end,value,sink);
				//	if(path_end!=nullptr)
				//		{
				//		fprintf(sink,"[%s] %zu\n",key.c_str(),level);
				//		}
					},eh);
				}

			template<Type t>
			auto& dataGet() noexcept
				{return dataGet<t>(m_data);}

			template<Type t>
			const auto& dataGet() const noexcept
				{return dataGet<t>(m_data);}

		private:
			template<class T>
			static constexpr bool equals_or(T x)
				{return 0;}

			template<class T,class ... U>
			static constexpr bool equals_or(T x,T y,U ... values)
				{return x==y || equals_or(x,values...);}

			template<class Cstr,class Stream,class... to_escape>
			static void write(Cstr src,Stream& stream,to_escape ... esc)
				{
				while(true)
					{
					auto ch_in=*src;
					if(ch_in=='\0')
						{return;}
					if(equals_or(ch_in,esc...))
						{putc('\\',stream);}
					putc(ch_in,stream);
					++src;
					}
				}

			template<Type t,class dummy=void>
			struct RecordWrite
				{
				template<class Sink,class KeyCstr,class Value,class ExceptionHandler>
				static void doIt(Sink& sink,size_t level,KeyCstr key,const Value& value,ExceptionHandler eh)
					{
					assert(key!=nullptr);
					assert(level!=0);
					for(size_t k=0;k<level-1;++k)
						{putc('\t',sink);}
					putc('"',sink);
					write(key,sink,'"');
					fprintf(sink,"\"%s:\n",type(t));
				//	write(value,sink);
					}
				};

			template<class dummy>
			struct RecordWrite<Type::COMPOUND,dummy>
				{
				template<class Sink,class KeyCstr,class Value,class ExceptionHandler>
				static void doIt(Sink& sink,size_t level,KeyCstr key,const Value& value,ExceptionHandler eh)
					{
					if(key!=nullptr)
						{						
						assert(level!=0);
						for(size_t k=0;k<level-1;++k)
							{putc('\t',sink);}
						putc('"',sink);
						write(key,sink,'"');
						fputs("\":\n",sink);
						}
					}
				};

			template<class dummy>
			struct RecordWrite<Type::COMPOUND_ARRAY,dummy>
				{
				template<class Sink,class KeyCstr,class Value,class ExceptionHandler>
				static void doIt(Sink& sink,size_t level,KeyCstr key,const Value& value,ExceptionHandler eh)
					{
					if(key!=nullptr)
						{						
						assert(level!=0);
						for(size_t k=0;k<level-1;++k)
							{putc('\t',sink);}
						putc('"',sink);
						write(key,sink,'"');
						fputs("\":\n",sink);
						}
					}
				};



			using Key=StringType;
{'fraction_non_alphanumeric': 0.15183684872188855, 'fraction_numerical': 0.0030730548959351867, 'mean_word_length': 3.754316069057105, 'pattern_counts': {'":': 9, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '48229', 'n_tokens_mistral': 3118, 'n_tokens_neox': 2439, 'n_words': 451}