<commit_before>use std::collections::HashMap;

pub struct Translator
{
	rules: HashMap<(char, char), char>,
}

impl Translator
{
	pub fn new() -> Translator
	{
		Translator
		{
			rules: get_default_rules(),
		}
	}

	// Returns translated characted and boolean value indicating that previous character should be erased.
	fn translate_char(&self, current : char, previous : char) -> (char, bool)
	{
		match self.rules.get(&(current, previous))
		{
			Some(result) => (*result, true),
			None => match self.rules.get(&(current, ' '))
			{
				Some(result) => (*result, false),
				None => (current, false),
			}
		}
	}

	pub fn translate(&self, text : &str) -> String
	{
		let mut result = String::with_capacity(text.len());

		let mut previous = ' ';
		for c in text.chars()
		{
			let (translated, erase_previous) = self.translate_char(c, previous);
			if erase_previous
			{
				result.pop();
			}<commit_msg>Spaces processing has been fixed<commit_after>use std::collections::HashMap;

pub struct Translator
{
	rules: HashMap<(char, char), char>,
}

impl Translator
{
	pub fn new() -> Translator
	{
		Translator
		{
			rules: get_default_rules(),
		}
	}

	// Returns translated characted and boolean value indicating that previous character should be erased.
	fn translate_char(&self, current : char, previous : char) -> (char, bool)
	{
		match self.rules.get(&(current, previous))
		{
			Some(result) => (*result, previous != ' '),
			None => match self.rules.get(&(current, ' '))
			{
				Some(result) => (*result, false),
				None => (current, false),
			}
		}
	}

	pub fn translate(&self, text : &str) -> String
	{
		let mut result = String::with_capacity(text.len());

		let mut previous = ' ';
		for c in text.chars()
		{
			let (translated, erase_previous) = self.translate_char(c, previous);
			if erase_previous
			{
				result.pop();
			}
{'fraction_non_alphanumeric': 0.15027027027027026, 'fraction_numerical': 0.0, 'mean_word_length': 3.5146341463414634, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5183763', 'n_tokens_mistral': 703, 'n_tokens_neox': 607, 'n_words': 162}