<commit_before>/****************************************************************************
**
** SpeechManager.cpp
**
** Copyright (C) May 2016 Aimed
**
** This program is free software; you can redistribute it and/or modify
** it under the terms of the GNU General Public License as published by
** the Free Software Foundation; either version 2 of the License, or
** (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU General Public License for more details.
**
*****************************************************************************
*/
//---------------------------------------------------------------------------
#include "stdafx.h"

TSpeechManager *SpeechManager = NULL;
//---------------------------------------------------------------------------
//--------------------------------TSpeechItem--------------------------------
//---------------------------------------------------------------------------
TSpeechItem::TSpeechItem()
: m_Code(0), m_Data(L""), m_CheckStart(false), m_CheckEnd(false)
{
}
//---------------------------------------------------------------------------
TSpeechItem::~TSpeechItem()
{
}
//---------------------------------------------------------------------------
//------------------------------TSpeechManager-------------------------------
//---------------------------------------------------------------------------
TSpeechManager::TSpeechManager()
{
}
//---------------------------------------------------------------------------
TSpeechManager::~TSpeechManager()
{
}
//---------------------------------------------------------------------------
/*!

@return true   
*/
bool TSpeechManager::LoadSpeech()
{
	TMappedHeader &file = FileManager.SpeechMul;

	while (!file.IsEof())
	{
		TSpeechItem item;
		item.Code = file.ReadWordBE();
		WORD len = file.ReadWordBE();
		wstring str = file.ReadUtf8String(len);

		const WCHAR *data = str.c_str();

		
		DFOR(i, str.length(), 1)
		{
			if (data[i])
			{
				if (data[i] == L'*')
				{
					item.CheckEnd = true;
					str.resize(i);
				}

				break;
			}
		}

		if (str.length() && *str.c_str() == L'*')
		{
			item.CheckStart = true;
			str.erase(str.begin());
		}

		my_map[str] = 1;
		item.Data = str;

		TPRINT("[0x%04X]=(len=%i, cs=%i, ce=%i) %s\n", item.Code, len, item.CheckStart, item.CheckEnd, ToString(str).c_str());
	}

	return true;
}
//---------------------------------------------------------------------------<commit_msg>Убрал кривой код. Дополнение к предидущему коммиту: добавил инклюд хешмапы в stdafx.h и обьявление поля мапы локалей-векторов в SpeechManager.h<commit_after>/****************************************************************************
**
** SpeechManager.cpp
**
** Copyright (C) May 2016 Aimed
**
** This program is free software; you can redistribute it and/or modify
** it under the terms of the GNU General Public License as published by
** the Free Software Foundation; either version 2 of the License, or
** (at your option) any later version.
**
** This program is distributed in the hope that it will be useful,
** but WITHOUT ANY WARRANTY; without even the implied warranty of
** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
** GNU General Public License for more details.
**
*****************************************************************************
*/
//---------------------------------------------------------------------------
#include "stdafx.h"

TSpeechManager *SpeechManager = NULL;
//---------------------------------------------------------------------------
//--------------------------------TSpeechItem--------------------------------
//---------------------------------------------------------------------------
TSpeechItem::TSpeechItem()
: m_Code(0), m_Data(L""), m_CheckStart(false), m_CheckEnd(false)
{
}
//---------------------------------------------------------------------------
TSpeechItem::~TSpeechItem()
{
}
//---------------------------------------------------------------------------
//------------------------------TSpeechManager-------------------------------
//---------------------------------------------------------------------------
TSpeechManager::TSpeechManager()
{
}
//---------------------------------------------------------------------------
TSpeechManager::~TSpeechManager()
{
}
//---------------------------------------------------------------------------
/*!

@return true   
*/
bool TSpeechManager::LoadSpeech()
{
	TMappedHeader &file = FileManager.SpeechMul;

	while (!file.IsEof())
	{
		TSpeechItem item;
		item.Code = file.ReadWordBE();
		WORD len = file.ReadWordBE();
		wstring str = file.ReadUtf8String(len);

		const WCHAR *data = str.c_str();

		
		DFOR(i, str.length(), 1)
		{
			if (data[i])
			{
				if (data[i] == L'*')
				{
					item.CheckEnd = true;
					str.resize(i);
				}

				break;
			}
		}

		if (str.length() && *str.c_str() == L'*')
		{
			item.CheckStart = true;
			str.erase(str.begin());
		}

		item.Data = str;

		//TPRINT("[0x%04X]=(len=%i, cs=%i, ce=%i) %s\n", item.Code, len, item.CheckStart, item.CheckEnd, ToString(str).c_str());
	}

	return true;
}
//---------------------------------------------------------------------------
{'fraction_non_alphanumeric': 0.45818521983161836, 'fraction_numerical': 0.004303086997193639, 'mean_word_length': 1.278772378516624, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1176412', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1307, 'n_words': 332}