<commit_before>#include <stingraykit/serialization/Serialization.h>
#include <stingraykit/serialization/SettingsValue.h>
#include <queue>

namespace stingray
{

	SettingsValue* ObjectIStream::Get(const std::string &name)
	{
		SettingsValueMap &map = _root->get<SettingsValueMap>();
		SettingsValueMap::iterator i = map.find(name);
		return (i != map.end())? i->second.get(): NULL;
	}

	void ObjectIStream::for_each(const function<void (SettingsValue&)> &func)
	{
		SettingsValueList &list = _root->get<SettingsValueList>();
		for(SettingsValueList::iterator i = list.begin(); i != list.end(); ++i)
		{
			func(**i);
		}
	}

	void ObjectIStream::for_each_kv(const function<void (const std::string &, SettingsValue&)> &func)
	{
		SettingsValueMap &map = _root->get<SettingsValueMap>();
		for(SettingsValueMap::iterator i = map.begin(); i != map.end(); ++i)
		{
			func(i->first, *i->second);
		}
	}

	s64 ObjectIStream::GetInt() { return _root->get<s64>(); }

	bool ObjectIStream::is_null() const
	{ return _root->contains<EmptyType>(); }

	bool ObjectIStream::is_array() const
	{ return _root->contains<SettingsValueList>(); }

	void ObjectIStream::deserialize(std::string &str)
	{ str = _root->get<std::string>(); }

	void ObjectIStream::deserialize(bool &value)
	{ value = _root->get<bool>(); }

	void ObjectIStream::deserialize(double &value)
	{ value = _root->get<double>(); }

	void ObjectIStream::deserialize(std::vector<u8> & data)
	{
		if (_root->contains<std::vector<u8> >())
			data = _root->get<std::vector<u8> >();
		else if (_root->contains<std::string>())
			UnpackBinaryEncoding(data, _root->get<std::string>());
		else
			STINGRAYKIT_THROW("invalid type for binary data value");
	}

	static inline u8 xdigit(char c) {
		if (c >= '0' && c <= '9')
			return c - '0';
		c |= 0x20; //lowercasing
		if (c >= 'a' && c <= 'f')
			return c - 'a' + 10;
		STINGRAYKIT_THROW(std::runtime_error(std::string("invalid hex digit") + c));
	}

	void ObjectIStream::UnpackBinaryEncoding(byte_array &data, const std::string& str) //sorry for that :(
	{
		if (str.size() < 4 || str.compare(0, 4, "hex=") != 0)
			STINGRAYKIT_THROW(std::runtime_error("invalid binary encoding"));

		data.resize((str.size() - 4) / 2);
		byte_array::iterator dst = data.begin();
		for(std::string::const_iterator i = str.begin() + 4; i != str.end(); )
		{
			u8 h = xdigit(*i++) << 4;
			if (i == str.end())
				STINGRAYKIT_THROW(std::runtime_error("half-byte in hex dump"));
			*dst++ = xdigit(*i++) | h;
		}
	}

}<commit_msg>removed sorry comment as it's irrelevant now<commit_after>#include <stingraykit/serialization/Serialization.h>
#include <stingraykit/serialization/SettingsValue.h>
#include <queue>

namespace stingray
{

	SettingsValue* ObjectIStream::Get(const std::string &name)
	{
		SettingsValueMap &map = _root->get<SettingsValueMap>();
		SettingsValueMap::iterator i = map.find(name);
		return (i != map.end())? i->second.get(): NULL;
	}

	void ObjectIStream::for_each(const function<void (SettingsValue&)> &func)
	{
		SettingsValueList &list = _root->get<SettingsValueList>();
		for(SettingsValueList::iterator i = list.begin(); i != list.end(); ++i)
		{
			func(**i);
		}
	}

	void ObjectIStream::for_each_kv(const function<void (const std::string &, SettingsValue&)> &func)
	{
		SettingsValueMap &map = _root->get<SettingsValueMap>();
		for(SettingsValueMap::iterator i = map.begin(); i != map.end(); ++i)
		{
			func(i->first, *i->second);
		}
	}

	s64 ObjectIStream::GetInt() { return _root->get<s64>(); }

	bool ObjectIStream::is_null() const
	{ return _root->contains<EmptyType>(); }

	bool ObjectIStream::is_array() const
	{ return _root->contains<SettingsValueList>(); }

	void ObjectIStream::deserialize(std::string &str)
	{ str = _root->get<std::string>(); }

	void ObjectIStream::deserialize(bool &value)
	{ value = _root->get<bool>(); }

	void ObjectIStream::deserialize(double &value)
	{ value = _root->get<double>(); }

	void ObjectIStream::deserialize(std::vector<u8> & data)
	{
		if (_root->contains<std::vector<u8> >())
			data = _root->get<std::vector<u8> >();
		else if (_root->contains<std::string>())
			UnpackBinaryEncoding(data, _root->get<std::string>());
		else
			STINGRAYKIT_THROW("invalid type for binary data value");
	}

	static inline u8 xdigit(char c) {
		if (c >= '0' && c <= '9')
			return c - '0';
		c |= 0x20; //lowercasing
		if (c >= 'a' && c <= 'f')
			return c - 'a' + 10;
		STINGRAYKIT_THROW(std::runtime_error(std::string("invalid hex digit") + c));
	}

	void ObjectIStream::UnpackBinaryEncoding(byte_array &data, const std::string& str)
	{
		if (str.size() < 4 || str.compare(0, 4, "hex=") != 0)
			STINGRAYKIT_THROW(std::runtime_error("invalid binary encoding"));

		data.resize((str.size() - 4) / 2);
		byte_array::iterator dst = data.begin();
		for(std::string::const_iterator i = str.begin() + 4; i != str.end(); )
		{
			u8 h = xdigit(*i++) << 4;
			if (i == str.end())
				STINGRAYKIT_THROW(std::runtime_error("half-byte in hex dump"));
			*dst++ = xdigit(*i++) | h;
		}
	}

}
{'fraction_non_alphanumeric': 0.19154873430336855, 'fraction_numerical': 0.009966115208291808, 'mean_word_length': 4.869005847953217, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '403155', 'n_tokens_mistral': 1948, 'n_tokens_neox': 1795, 'n_words': 359}