<commit_before>#include <cstddef>  // NULL
#include <stdio.h>
#include <algorithm>
#include <stdexcept>
#include <iostream>

#include "TPLexPrefixTree.hh"

using namespace std;

int safe_tolower(int c)
{
	if (c == '')
		return '';
	else if (c == '')
		return '';
	else if (c == '')
		return '';
	return tolower(c);
}

int safe_toupper(int c)
{
	if (c == '')
		return '';
	else if (c == '')
		return '';
	else if (c == '')
		return '';
	return toupper(c);
}

TPLexPrefixTree::TPLexPrefixTree(std::map<std::string, int> &hmm_map,
		std::vector<Hmm> &hmms) :
	m_words(0), m_verbose(0), m_lm_lookahead(0), m_silence_is_word(true),
			m_hmm_map(hmm_map), m_hmms(hmms)
{
	initialize_nodes();
	m_lm_buf_count = 0;
	m_cross_word_triphones = true;
	m_optional_short_silence = true;
	m_short_silence_state = NULL;
	m_word_boundary_id = -1;
	m_ignore_case = false;
}

struct delete_node
{
	void operator()(TPLexPrefixTree::Node * ptr)
	{
		if (ptr != NULL)
			delete ptr;
	}
};

TPLexPrefixTree::~TPLexPrefixTree()
{
	for_each(m_nodes.begin(), m_nodes.end(), delete_node());
	m_nodes.clear();
}

void TPLexPrefixTree::set_lm_lookahead(int lm_lookahead)
{
	if (m_silence_node != NULL) {
		cerr << "WARNING: TPLexPrefixTree::set_lm_lookahead() called after reading lexicon." << endl;
		cerr << "WARNING: Lookahead setting will not be apply to the existing lexicon." << endl;
	}
	m_lm_lookahead = lm_lookahead;
}

void TPLexPrefixTree::initialize_lex_tree(void)
{
	m_words = 0;
	initialize_nodes();
	//	m_lm_lookahead = 0;

	m_lm_buf_count = 0;

	m_short_silence_state = NULL;
	m_word_boundary_id = -1;

	free_cross_word_network_connection_points();
	m_silence_arcs.clear();

	if (m_cross_word_triphones)
		create_cross_word_network();
}

void TPLexPrefixTree::add_word(std::vector<Hmm*> &hmm_list, int word_id)
{
	int i, j, k;
	node_vector hmm_state_nodes;
	node_vector source_nodes, sink_nodes;
	std::vector<float> source_trans_log_probs;
	std::vector<float> sink_trans_log_probs;
	int word_end;
	Arc end_arc;
	bool silence = false;

	// NOTE! HMM states are indexed with respect to their mixture model.
	// This is a quick solution to allow tied HMM states. The right way
	// to do this would be to load HMMs so that different HMMs would
	// really share the same states. Now it is merely assumed that the
	// (local) transitions in the states with the same mixtures are the same.

	// Invariants: source_nodes.size() == source_trans_log_probs.size(),
	//             sink_nodes.size() == sink_trans_log_probs.size()

	if (hmm_list.size() == 1 && hmm_list[0]->states.size() == 3
			&& hmm_list[0]->label == "_") {
		// Short silence
		m_short_silence_state = &hmm_list[0]->state(2);
		// Check the self transition
		assert( m_short_silence_state->transitions.size() == 2 &&
				m_short_silence_state->transitions[0].target == 2 );
		return;
	}

	if (hmm_list.size() == 1 && hmm_list[0]->label == "__")
		silence = true;

	source_nodes.push_back(m_root_node);
	source_trans_log_probs.push_back(0);
	word_end = -1;
	for (i = 0; i < hmm_list.size(); i++) {
		assert( source_nodes.size() > 0 );

		if (i == hmm_list.size() - 1) // Last node
		{
			// Added a check that the last phone is a triphone (size == 3).
			//   2012-05-07 / SE
			if (m_cross_word_triphones && !silence && (hmm_list[i]->label.size() == 3)) {
				// Link to cross word network.
				// First, add a null node with the word ID.
				Node *wid_node = new Node(word_id);
				Arc temp_arc;
				std::vector<Arc> new_arcs;
				wid_node->node_id = m_nodes.size();
				wid_node->flags = NODE_USE_WORD_END_BEAM;
				if (i == 0)
					wid_node->flags |= NODE_FIRST_STATE_OF_WORD;
				m_nodes.push_back(wid_node);
				temp_arc.next = wid_node;
				if (i == 0)
					wid_node->flags |= NODE_FIRST_STATE_OF_WORD;
				for (j = 0; j < source_nodes.size(); j++) {
					temp_arc.log_prob = source_trans_log_probs[j];
					source_nodes[j]->arcs.push_back(temp_arc);
				}

				// Link null node to cross word network
				std::string temp1(hmm_list[i]->label, 0, 1);
				std::string temp2(hmm_list[i]->label, 2, 1);
				std::string key = temp1 + temp2;
				link_node_to_fan_network(key, new_arcs, true, false, 0);
				for (j = 0; j < new_arcs.size(); j++)
					wid_node->arcs.push_back(new_arcs[j]);
				if (i == 0) {
					// If word has only one HMM, make another instance of the word
					// inside the cross word network.
					add_single_hmm_word_for_cross_word_modeling(hmm_list[0],
							word_id);
				}
				else if (i == 1) {
					// Word has two HMM states, mark the null node as a connection
					// point for linking from cross word network
					add_fan_in_connection_node(wid_node, hmm_list[0]->label);
				}
				source_nodes.clear();
				source_trans_log_probs.clear();
				break; // Skip last state (handled in the cross word network).
			}
			if (!silence || m_silence_is_word)
				word_end = word_id;
		}

		// Set up a table for storing the nodes corresponding to the HMM states
		hmm_state_nodes.clear();
		hmm_state_nodes.insert(hmm_state_nodes.end(),
				hmm_list[i]->states.size() - 2, NULL);
		sink_nodes.clear();
		sink_trans_log_probs.clear();

		// Expand previous sink nodes with new source transitions
		for (j = 0; j < source_nodes.size(); j++) // Iterate previous sink nodes
		{
			for (k = 0; k < hmm_list[i]->state(0).transitions.size(); k++) {
				expand_lexical_tree(source_nodes[j], hmm_list[i],
						hmm_list[i]->state(0).transitions[k],
						source_trans_log_probs[j], word_end, hmm_state_nodes,
						sink_nodes, sink_trans_log_probs, 0);
			}
		}

		if (m_cross_word_triphones && i == 1 && hmm_list.size() > 2) {
			// Mark the state for linking cross word network back to the
			// lexical tree.
			add_fan_in_connection_node(hmm_state_nodes[0], hmm_list[0]->label);
		}
		if (silence && i == 0) {
			m_silence_node = hmm_state_nodes[0];
			m_silence_node->flags |= NODE_SILENCE_FIRST;
			if (m_silence_is_word)
				m_silence_node->flags |= NODE_FIRST_STATE_OF_WORD;
		}
		else if (i == 0)
			hmm_state_nodes[0]->flags |= NODE_FIRST_STATE_OF_WORD;

		// Expand other states, from left to right
		for (j = 2; j < hmm_list[i]->states.size(); j++) {
			assert( hmm_state_nodes[j-2] != NULL );
			for (k = 0; k < hmm_list[i]->state(j).transitions.size(); k++) {
				expand_lexical_tree(hmm_state_nodes[j - 2], hmm_list[i],<commit_msg>Added a check that the first phone is a triphone, before adding fan in connection node.<commit_after>#include <cstddef>  // NULL
#include <stdio.h>
#include <algorithm>
#include <stdexcept>
#include <iostream>

#include "TPLexPrefixTree.hh"

using namespace std;

int safe_tolower(int c)
{
	if (c == '')
		return '';
	else if (c == '')
		return '';
	else if (c == '')
		return '';
	return tolower(c);
}

int safe_toupper(int c)
{
	if (c == '')
		return '';
	else if (c == '')
		return '';
	else if (c == '')
		return '';
	return toupper(c);
}

TPLexPrefixTree::TPLexPrefixTree(std::map<std::string, int> &hmm_map,
		std::vector<Hmm> &hmms) :
	m_words(0), m_verbose(0), m_lm_lookahead(0), m_silence_is_word(true),
			m_hmm_map(hmm_map), m_hmms(hmms)
{
	initialize_nodes();
	m_lm_buf_count = 0;
	m_cross_word_triphones = true;
	m_optional_short_silence = true;
	m_short_silence_state = NULL;
	m_word_boundary_id = -1;
	m_ignore_case = false;
}

struct delete_node
{
	void operator()(TPLexPrefixTree::Node * ptr)
	{
		if (ptr != NULL)
			delete ptr;
	}
};

TPLexPrefixTree::~TPLexPrefixTree()
{
	for_each(m_nodes.begin(), m_nodes.end(), delete_node());
	m_nodes.clear();
}

void TPLexPrefixTree::set_lm_lookahead(int lm_lookahead)
{
	if (m_silence_node != NULL) {
		cerr << "WARNING: TPLexPrefixTree::set_lm_lookahead() called after reading lexicon." << endl;
		cerr << "WARNING: Lookahead setting will not be apply to the existing lexicon." << endl;
	}
	m_lm_lookahead = lm_lookahead;
}

void TPLexPrefixTree::initialize_lex_tree(void)
{
	m_words = 0;
	initialize_nodes();
	//	m_lm_lookahead = 0;

	m_lm_buf_count = 0;

	m_short_silence_state = NULL;
	m_word_boundary_id = -1;

	free_cross_word_network_connection_points();
	m_silence_arcs.clear();

	if (m_cross_word_triphones)
		create_cross_word_network();
}

void TPLexPrefixTree::add_word(std::vector<Hmm*> &hmm_list, int word_id)
{
	int i, j, k;
	node_vector hmm_state_nodes;
	node_vector source_nodes, sink_nodes;
	std::vector<float> source_trans_log_probs;
	std::vector<float> sink_trans_log_probs;
	int word_end;
	Arc end_arc;
	bool silence = false;

	// NOTE! HMM states are indexed with respect to their mixture model.
	// This is a quick solution to allow tied HMM states. The right way
	// to do this would be to load HMMs so that different HMMs would
	// really share the same states. Now it is merely assumed that the
	// (local) transitions in the states with the same mixtures are the same.

	// Invariants: source_nodes.size() == source_trans_log_probs.size(),
	//             sink_nodes.size() == sink_trans_log_probs.size()

	if (hmm_list.size() == 1 && hmm_list[0]->states.size() == 3
			&& hmm_list[0]->label == "_") {
		// Short silence
		m_short_silence_state = &hmm_list[0]->state(2);
		// Check the self transition
		assert( m_short_silence_state->transitions.size() == 2 &&
				m_short_silence_state->transitions[0].target == 2 );
		return;
	}

	if (hmm_list.size() == 1 && hmm_list[0]->label == "__")
		silence = true;

	source_nodes.push_back(m_root_node);
	source_trans_log_probs.push_back(0);
	word_end = -1;
	for (i = 0; i < hmm_list.size(); i++) {
		assert( source_nodes.size() > 0 );

		if (i == hmm_list.size() - 1) // Last node
		{
			// Added a check that the last phone is a triphone (label is five characters).
			//   2012-05-07 / SE
			if (m_cross_word_triphones && !silence && (hmm_list[i]->label.size() == 5)) {
				// Link to cross word network.
				// First, add a null node with the word ID.
				Node *wid_node = new Node(word_id);
				Arc temp_arc;
				std::vector<Arc> new_arcs;
				wid_node->node_id = m_nodes.size();
				wid_node->flags = NODE_USE_WORD_END_BEAM;
				if (i == 0)
					wid_node->flags |= NODE_FIRST_STATE_OF_WORD;
				m_nodes.push_back(wid_node);
				temp_arc.next = wid_node;
				if (i == 0)
					wid_node->flags |= NODE_FIRST_STATE_OF_WORD;
				for (j = 0; j < source_nodes.size(); j++) {
					temp_arc.log_prob = source_trans_log_probs[j];
					source_nodes[j]->arcs.push_back(temp_arc);
				}

				// Link null node to cross word network
				std::string temp1(hmm_list[i]->label, 0, 1);
				std::string temp2(hmm_list[i]->label, 2, 1);
				std::string key = temp1 + temp2;
				link_node_to_fan_network(key, new_arcs, true, false, 0);
				for (j = 0; j < new_arcs.size(); j++)
					wid_node->arcs.push_back(new_arcs[j]);
				if (i == 0) {
					// If word has only one HMM, make another instance of the word
					// inside the cross word network.
					add_single_hmm_word_for_cross_word_modeling(hmm_list[0],
							word_id);
				}
				// Added a check that the first phone is a triphone (label is five characters).
				//   2012-05-07 / SE
				else if ((i == 1) && (hmm_list[0]->label.size() == 5)) {
					// Word has two HMMs, mark the null node as a connection
					// point for linking from cross word network
					add_fan_in_connection_node(wid_node, hmm_list[0]->label);
				}
				source_nodes.clear();
				source_trans_log_probs.clear();
				break; // Skip last state (handled in the cross word network).
			}
			if (!silence || m_silence_is_word)
				word_end = word_id;
		}

		// Set up a table for storing the nodes corresponding to the HMM states
		hmm_state_nodes.clear();
		hmm_state_nodes.insert(hmm_state_nodes.end(),
				hmm_list[i]->states.size() - 2, NULL);
		sink_nodes.clear();
		sink_trans_log_probs.clear();

		// Expand previous sink nodes with new source transitions
		for (j = 0; j < source_nodes.size(); j++) // Iterate previous sink nodes
		{
			for (k = 0; k < hmm_list[i]->state(0).transitions.size(); k++) {
				expand_lexical_tree(source_nodes[j], hmm_list[i],
						hmm_list[i]->state(0).transitions[k],
						source_trans_log_probs[j], word_end, hmm_state_nodes,
						sink_nodes, sink_trans_log_probs, 0);
			}
		}

		// Added a check that the first phone is a triphone (label is five characters).
		//   2012-05-07 / SE
		if (m_cross_word_triphones && (i == 1) && (hmm_list.size() > 2) && (hmm_list[0]->label.size() == 5)) {
			// Mark the state for linking cross word network back to the
			// lexical tree.
			add_fan_in_connection_node(hmm_state_nodes[0], hmm_list[0]->label);
		}
		if (silence && i == 0) {
			m_silence_node = hmm_state_nodes[0];
			m_silence_node->flags |= NODE_SILENCE_FIRST;
			if (m_silence_is_word)
				m_silence_node->flags |= NODE_FIRST_STATE_OF_WORD;
		}
		else if (i == 0)
			hmm_state_nodes[0]->flags |= NODE_FIRST_STATE_OF_WORD;

		// Expand other states, from left to right
		for (j = 2; j < hmm_list[i]->states.size(); j++) {
			assert( hmm_state_nodes[j-2] != NULL );
			for (k = 0; k < hmm_list[i]->state(j).transitions.size(); k++) {
				expand_lexical_tree(hmm_state_nodes[j - 2], hmm_list[i],
{'fraction_non_alphanumeric': 0.14121814121814122, 'fraction_numerical': 0.012243012243012243, 'mean_word_length': 3.7732451304667403, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5794733', 'n_tokens_mistral': 5597, 'n_tokens_neox': 4865, 'n_words': 1149}