<commit_before>#pragma once


#include "core/default_allocator.h"
#include "core/lux.h"
#include <cstring>


namespace Lux
{
	

template <class T, typename Allocator = DefaultAllocator>
class base_string
{
	public:
		static base_string<T, Allocator> create(unsigned int length, const char *s)
		{
			return base_string<T, Allocator>(s);
		}

		base_string(const Allocator& allocator)
			: m_allocator(allocator)
		{
			m_cstr = NULL;
			m_size = 0;
		}

		base_string()
		{
			m_cstr = NULL;
			m_size = 0;
		}

		base_string(const base_string<T, Allocator>& rhs, int start, size_t length)
		{
			m_size = length - start <= rhs.m_size ? length : rhs.m_size - start;
			m_cstr = (T*)m_allocator.allocate((m_size + 1) * sizeof(T));
			memcpy(m_cstr, rhs.m_cstr + start, m_size * sizeof(T));
			m_cstr[m_size] = 0;
		}
		
		base_string(const base_string<T, Allocator>& rhs)
		{
			m_cstr = (T*)m_allocator.allocate((rhs.m_size + 1) * sizeof(T));
			m_size = rhs.m_size;
			memcpy(m_cstr, rhs.m_cstr, m_size * sizeof(T));
			m_cstr[m_size] = 0;
		}

		explicit base_string(const T* rhs)
		{
			m_size = strlen(rhs);
			m_cstr = (T*)m_allocator.allocate((m_size + 1) * sizeof(T));
			memcpy(m_cstr, rhs, sizeof(T) * (m_size + 1));
		}

		~base_string()
		{
			m_allocator.deallocate(m_cstr);
		}

		void operator = (const base_string<T, Allocator>& rhs) 
		{
			if(&rhs != this)
			{
				m_allocator.deallocate(m_cstr);
				m_cstr = (T*)m_allocator.allocate((rhs.m_size + 1) * sizeof(T));
				m_size = rhs.m_size;
				memcpy(m_cstr, rhs.m_cstr, sizeof(T) * (m_size + 1));
			}
		}

		void operator = (const T* rhs) 
		{
			m_allocator.deallocate(m_cstr);
			m_size = strlen(rhs);
			m_cstr = (T*)m_allocator.allocate((m_size + 1) * sizeof(T));
			memcpy(m_cstr, rhs, sizeof(T) * (m_size + 1));
		}

		bool operator !=(const base_string<T, Allocator>& rhs) const
		{
			return this->strcmp(rhs.m_cstr) != 0;
		}

		bool operator !=(const T* rhs) const
		{
			return this->strcmp(rhs) != 0;
		}

		bool operator ==(const base_string<T, Allocator>& rhs) const
		{
			return this->strcmp(rhs.m_cstr) == 0;
		}

		bool operator ==(const T* rhs) const
		{
			return this->strcmp(rhs) == 0;
		}

		bool operator <(const base_string<T, Allocator>& rhs) const
		{
			return this->strcmp(rhs.m_cstr) < 0;
		}

		bool operator >(const base_string<T, Allocator>& rhs) const
		{
			return this->strcmp(rhs.m_cstr) > 0;
		}
		
		int rfind(T c) const
		{
			int i = m_size - 1;
			while(i >= 0 && m_cstr[i] != c)
			{
				--i;
			}
			return i >= 0 ? i : npos;
		}

		int length() const { return m_size; }

		const T* c_str() const { return m_cstr; }
		
		base_string<T, Allocator> substr(int start, int length) const
		{
			return base_string<T, Allocator>(*this, start, length);
		}
		
		void operator += (const T* rhs)
		{
			if(m_cstr)
			{
				m_size += base_string<T>::strlen(rhs);
				m_cstr = (T*)m_allocator.reallocate(m_cstr, m_size + 1);
		        base_string<T>::strcat(m_cstr, rhs);			
			}
			else
			{
				m_size = base_string<T>::strlen(rhs);
				m_cstr = (T*)m_allocator.allocate(m_size + 1);
				base_string<T>::strcpy(m_cstr, rhs);
			}
		}

		void operator += (const base_string<T, Allocator>& rhs)
		{
			if(!rhs.m_cstr)
			{
				return;
			}
			if(m_cstr)
			{
				m_size += rhs.length();
				m_cstr = (T*)m_allocator.reallocate(m_cstr, m_size + 1);
				base_string<T>::strcat(m_cstr, rhs.m_cstr);
			}
			else
			{
				*this = rhs;
			}
		}

		void insert(size_t pos, T value)
		{
			m_cstr = (T*)m_allocator.reallocate(m_cstr, m_size + 2);
			for(size_t i = m_size; i > pos; ++i)
			{
				m_cstr[i] = m_cstr[i-1];
			}
			m_cstr[pos] = value;
			++m_size;
		}<commit_msg>base_string<T>::insert fixed<commit_after>#pragma once


#include "core/default_allocator.h"
#include "core/lux.h"
#include <cstring>


namespace Lux
{
	

template <class T, typename Allocator = DefaultAllocator>
class base_string
{
	public:
		static base_string<T, Allocator> create(unsigned int length, const char *s)
		{
			return base_string<T, Allocator>(s);
		}

		base_string(const Allocator& allocator)
			: m_allocator(allocator)
		{
			m_cstr = NULL;
			m_size = 0;
		}

		base_string()
		{
			m_cstr = NULL;
			m_size = 0;
		}

		base_string(const base_string<T, Allocator>& rhs, int start, size_t length)
		{
			m_size = length - start <= rhs.m_size ? length : rhs.m_size - start;
			m_cstr = (T*)m_allocator.allocate((m_size + 1) * sizeof(T));
			memcpy(m_cstr, rhs.m_cstr + start, m_size * sizeof(T));
			m_cstr[m_size] = 0;
		}
		
		base_string(const base_string<T, Allocator>& rhs)
		{
			m_cstr = (T*)m_allocator.allocate((rhs.m_size + 1) * sizeof(T));
			m_size = rhs.m_size;
			memcpy(m_cstr, rhs.m_cstr, m_size * sizeof(T));
			m_cstr[m_size] = 0;
		}

		explicit base_string(const T* rhs)
		{
			m_size = strlen(rhs);
			m_cstr = (T*)m_allocator.allocate((m_size + 1) * sizeof(T));
			memcpy(m_cstr, rhs, sizeof(T) * (m_size + 1));
		}

		~base_string()
		{
			m_allocator.deallocate(m_cstr);
		}

		void operator = (const base_string<T, Allocator>& rhs) 
		{
			if(&rhs != this)
			{
				m_allocator.deallocate(m_cstr);
				m_cstr = (T*)m_allocator.allocate((rhs.m_size + 1) * sizeof(T));
				m_size = rhs.m_size;
				memcpy(m_cstr, rhs.m_cstr, sizeof(T) * (m_size + 1));
			}
		}

		void operator = (const T* rhs) 
		{
			m_allocator.deallocate(m_cstr);
			m_size = strlen(rhs);
			m_cstr = (T*)m_allocator.allocate((m_size + 1) * sizeof(T));
			memcpy(m_cstr, rhs, sizeof(T) * (m_size + 1));
		}

		bool operator !=(const base_string<T, Allocator>& rhs) const
		{
			return this->strcmp(rhs.m_cstr) != 0;
		}

		bool operator !=(const T* rhs) const
		{
			return this->strcmp(rhs) != 0;
		}

		bool operator ==(const base_string<T, Allocator>& rhs) const
		{
			return this->strcmp(rhs.m_cstr) == 0;
		}

		bool operator ==(const T* rhs) const
		{
			return this->strcmp(rhs) == 0;
		}

		bool operator <(const base_string<T, Allocator>& rhs) const
		{
			return this->strcmp(rhs.m_cstr) < 0;
		}

		bool operator >(const base_string<T, Allocator>& rhs) const
		{
			return this->strcmp(rhs.m_cstr) > 0;
		}
		
		int rfind(T c) const
		{
			int i = m_size - 1;
			while(i >= 0 && m_cstr[i] != c)
			{
				--i;
			}
			return i >= 0 ? i : npos;
		}

		int length() const { return m_size; }

		const T* c_str() const { return m_cstr; }
		
		base_string<T, Allocator> substr(int start, int length) const
		{
			return base_string<T, Allocator>(*this, start, length);
		}
		
		void operator += (const T* rhs)
		{
			if(m_cstr)
			{
				m_size += base_string<T>::strlen(rhs);
				m_cstr = (T*)m_allocator.reallocate(m_cstr, m_size + 1);
		        base_string<T>::strcat(m_cstr, rhs);			
			}
			else
			{
				m_size = base_string<T>::strlen(rhs);
				m_cstr = (T*)m_allocator.allocate(m_size + 1);
				base_string<T>::strcpy(m_cstr, rhs);
			}
		}

		void operator += (const base_string<T, Allocator>& rhs)
		{
			if(!rhs.m_cstr)
			{
				return;
			}
			if(m_cstr)
			{
				m_size += rhs.length();
				m_cstr = (T*)m_allocator.reallocate(m_cstr, m_size + 1);
				base_string<T>::strcat(m_cstr, rhs.m_cstr);
			}
			else
			{
				*this = rhs;
			}
		}

		void insert(size_t pos, T value)
		{
			m_cstr = (T*)m_allocator.reallocate(m_cstr, m_size + 2);
			for(size_t i = m_size + 1; i > pos; --i)
			{
				m_cstr[i] = m_cstr[i-1];
			}
			m_cstr[pos] = value;
			++m_size;
		}
{'fraction_non_alphanumeric': 0.1591803501153481, 'fraction_numerical': 0.007192292034197313, 'mean_word_length': 3.0944444444444446, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1953723', 'n_tokens_mistral': 3709, 'n_tokens_neox': 3142, 'n_words': 604}