<commit_before>//
//  lr_parser_stage.cpp
//  Parse
//
//  Created by <NAME> on 27/08/2011.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#include <sstream>
#include "TameParse/Compiler/lr_parser_stage.h"
#include "TameParse/Lr/conflict.h"
#include "TameParse/Lr/ignored_symbols.h"
#include "TameParse/Language/formatter.h"
#include "TameParse/Lr/ast_parser.h"

using namespace std;
using namespace dfa;
using namespace contextfree;
using namespace lr;
using namespace language;
using namespace compiler;

/// \brief Constructor
lr_parser_stage::lr_parser_stage(console_container& console, const std::wstring& filename, language_stage* languageCompiler, lexer_stage* lexerCompiler, const vector<wstring>& startSymbols)
: compilation_stage(console, filename)
, m_Language(languageCompiler)
, m_LexerCompiler(lexerCompiler)
, m_StartSymbols(startSymbols)
, m_StartPosition(position(-1,-1,-1))
, m_Parser(NULL)
, m_Tables(NULL) {
	// Add empty positions for each symbol
	for (size_t x=0; x<m_StartSymbols.size(); x++) {
		m_SymbolStartPosition.push_back(position(-1,-1,-1));
	}
}

/// \brief Constructure which builds the list of start symbols from a parser block
lr_parser_stage::lr_parser_stage(console_container& console, const std::wstring& filename, language_stage* languageCompiler, lexer_stage* lexerCompiler, parser_block* parserBlock) 
: compilation_stage(console, filename)
, m_Language(languageCompiler)
, m_LexerCompiler(lexerCompiler)
, m_StartPosition(parserBlock->start_pos())
, m_StartSymbols(parserBlock->start_symbols())
, m_Parser(NULL)
, m_Tables(NULL) {
	// Make all the symbols begin in the same place as this block
	// TODO: actually record where the symbols are specified
	for (size_t x=0; x<m_StartSymbols.size(); x++) {
		m_SymbolStartPosition.push_back(m_StartPosition);
	}	
}

/// \brief Destructor
lr_parser_stage::~lr_parser_stage() {
	// Finished with the parser
	if (m_Parser) {
		delete m_Parser;
		m_Parser = NULL;
	}
    
    if (m_Tables) {
        delete m_Tables;
        m_Tables = NULL;
    }
}


/// \brief Search for clashing guards
///
/// A clashing guard occurs when two different guards are specified for the same symbol
/// for a state. These need to be warned on as they can be an indication that the
/// parser is invalid (ie, the parser will not distinguish something that it is
/// supposed to distinguish)
static void warn_clashing_guards(console& cons, const language_stage* language, lalr_builder* builder) {
	// TODO: have a way to mark guards that are permitted to be in the same state
	
	// Iterate through the states
	for (int stateId = 0; stateId < builder->count_states(); stateId++) {
		// Map of terminal symbol IDs to guard actions
		typedef map<item_container, set<item_container> > guard_to_symbol;
		guard_to_symbol guardForSymbol;

		// Search for guard actions 
		const lr_action_set& actions = builder->actions_for_state(stateId);

		for (lr_action_set::const_iterator nextAction = actions.begin(); nextAction != actions.end(); nextAction++) {
			// Ignore actions that are not guard actions
			if ((*nextAction)->type() != lr_action::act_guard) continue;

			// Ignore the action if it's not on a terminal symbol
			if ((*nextAction)->item()->type() != item::terminal) continue;

			// The guard item is the nonterminal of the action rule
			guardForSymbol[(*nextAction)->item()].insert((*nextAction)->rule()->nonterminal());
		}

		// Warn of any guards that have conflicting actions
		set<item_container> warnedGuards;

		for (guard_to_symbol::iterator guarded = guardForSymbol.begin(); guarded != guardForSymbol.end(); guarded++) {
			// No clash if there's only one guard for this symbol
			if (guarded->second.size() <= 1) continue;

			// Ignore guards that we've already warned about
			bool warned = true;
			for (set<item_container>::iterator it = guarded->second.begin(); it != guarded->second.end(); it++) {
				if (warnedGuards.find(*it) == warnedGuards.end()) {
					// Haven't warned about this guard yet, so this is a new kind of clash
					warned = false;
					break;
				}
			}

			if (warned) continue;

			// Default severity is 'warning' for the first item, and 'detail' for the remainder
			bool shownWarning = false;

			// Get the state ID
			const lalr_state_container& state = builder->machine().state_with_id(stateId);

			// Iterate through the guards
			for (set<item_container>::iterator clashingGuard = guarded->second.begin(); clashingGuard != guarded->second.end(); clashingGuard++) {
				// Warned about this guard
				warnedGuards.insert(*clashingGuard);

				// Find all of the LR items that have a reference to this guard
				for (lalr_state::iterator lrItem = state->begin(); lrItem != state->end(); lrItem++) {
					// Ignore at end items
					if ((*lrItem)->at_end()) continue;

					// Ignore items that do not refer to this guard
					// (Assumption is that all usages of the guard have the same initial set, so any symbol refers to any reference to this guard)
					if ((*lrItem)->rule()->items()[(*lrItem)->offset()] != *clashingGuard) continue;

					// Get the position of this rule
					int             ruleId      = (*lrItem)->rule()->identifier(*language->grammar());
					position        rulePos     = language->rule_definition_pos(ruleId);
                    const wstring&  ruleFile    = language->rule_definition_file(ruleId);

					// We know the item, the guard and the position: generate the error message
					if (!shownWarning) {
						wstringstream warning;
						warning	<< L"Found clashing guards on symbol '" 
								<< formatter::to_string(*guarded->first, builder->gram(), builder->terminals())
								<< L"'";
						
						cons.report_error(error(error::sev_warning, ruleFile, L"CLASHING_GUARDS", warning.str(), rulePos));
					}

					// Provide detail messages on where the clash is
					wstringstream msg;
					msg << (!shownWarning?L"when used here: ":L"or here: ")
						<< formatter::to_string(**lrItem, builder->gram(), builder->terminals());
					
					cons.report_error(error(error::sev_detail, language->filename(), L"CLASHING_GUARDS_DETAIL", msg.str(), rulePos));

					shownWarning = true;
				}
			}
		}
	}
}


/// \brief Compiles the parser specified by the parameters to this stage
void lr_parser_stage::compile() {
    // Verbose message to say which stage we're at
	cons().verbose_stream() << L"  = Building parser" << endl;

	// Recycle the parser generator if it already exists
	if (m_Parser) {
		delete m_Parser;
		m_Parser = NULL;
	}
    
    if (m_Tables) {
        delete m_Tables;
        m_Tables = NULL;
    }

	// Sanity check (language)
	if (!m_Language) {
		cons().report_error(error(error::sev_bug, filename(), L"BUG_NO_LANGUAGE", L"Language compiler stage was not supplied to parser stage", m_StartPosition));
		return;
	}

	if (!m_Language->ndfa()) {
		cons().report_error(error(error::sev_bug, filename(), L"BUG_NO_LANGUAGE_NDFA", L"Language compiler stage has not generated a lexer", m_StartPosition));
		return;
	}

	if (!m_Language->terminals()) {
		cons().report_error(error(error::sev_bug, filename(), L"BUG_NO_LANGUAGE_TERMINALS", L"Language compiler stage has not generated a terminal dictionary", m_StartPosition));		<commit_msg>Changed NDFA check to a lexer check<commit_after>//
//  lr_parser_stage.cpp
//  Parse
//
//  Created by <NAME> on 27/08/2011.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#include <sstream>
#include "TameParse/Compiler/lr_parser_stage.h"
#include "TameParse/Lr/conflict.h"
#include "TameParse/Lr/ignored_symbols.h"
#include "TameParse/Language/formatter.h"
#include "TameParse/Lr/ast_parser.h"

using namespace std;
using namespace dfa;
using namespace contextfree;
using namespace lr;
using namespace language;
using namespace compiler;

/// \brief Constructor
lr_parser_stage::lr_parser_stage(console_container& console, const std::wstring& filename, language_stage* languageCompiler, lexer_stage* lexerCompiler, const vector<wstring>& startSymbols)
: compilation_stage(console, filename)
, m_Language(languageCompiler)
, m_LexerCompiler(lexerCompiler)
, m_StartSymbols(startSymbols)
, m_StartPosition(position(-1,-1,-1))
, m_Parser(NULL)
, m_Tables(NULL) {
	// Add empty positions for each symbol
	for (size_t x=0; x<m_StartSymbols.size(); x++) {
		m_SymbolStartPosition.push_back(position(-1,-1,-1));
	}
}

/// \brief Constructure which builds the list of start symbols from a parser block
lr_parser_stage::lr_parser_stage(console_container& console, const std::wstring& filename, language_stage* languageCompiler, lexer_stage* lexerCompiler, parser_block* parserBlock) 
: compilation_stage(console, filename)
, m_Language(languageCompiler)
, m_LexerCompiler(lexerCompiler)
, m_StartPosition(parserBlock->start_pos())
, m_StartSymbols(parserBlock->start_symbols())
, m_Parser(NULL)
, m_Tables(NULL) {
	// Make all the symbols begin in the same place as this block
	// TODO: actually record where the symbols are specified
	for (size_t x=0; x<m_StartSymbols.size(); x++) {
		m_SymbolStartPosition.push_back(m_StartPosition);
	}	
}

/// \brief Destructor
lr_parser_stage::~lr_parser_stage() {
	// Finished with the parser
	if (m_Parser) {
		delete m_Parser;
		m_Parser = NULL;
	}
    
    if (m_Tables) {
        delete m_Tables;
        m_Tables = NULL;
    }
}


/// \brief Search for clashing guards
///
/// A clashing guard occurs when two different guards are specified for the same symbol
/// for a state. These need to be warned on as they can be an indication that the
/// parser is invalid (ie, the parser will not distinguish something that it is
/// supposed to distinguish)
static void warn_clashing_guards(console& cons, const language_stage* language, lalr_builder* builder) {
	// TODO: have a way to mark guards that are permitted to be in the same state
	
	// Iterate through the states
	for (int stateId = 0; stateId < builder->count_states(); stateId++) {
		// Map of terminal symbol IDs to guard actions
		typedef map<item_container, set<item_container> > guard_to_symbol;
		guard_to_symbol guardForSymbol;

		// Search for guard actions 
		const lr_action_set& actions = builder->actions_for_state(stateId);

		for (lr_action_set::const_iterator nextAction = actions.begin(); nextAction != actions.end(); nextAction++) {
			// Ignore actions that are not guard actions
			if ((*nextAction)->type() != lr_action::act_guard) continue;

			// Ignore the action if it's not on a terminal symbol
			if ((*nextAction)->item()->type() != item::terminal) continue;

			// The guard item is the nonterminal of the action rule
			guardForSymbol[(*nextAction)->item()].insert((*nextAction)->rule()->nonterminal());
		}

		// Warn of any guards that have conflicting actions
		set<item_container> warnedGuards;

		for (guard_to_symbol::iterator guarded = guardForSymbol.begin(); guarded != guardForSymbol.end(); guarded++) {
			// No clash if there's only one guard for this symbol
			if (guarded->second.size() <= 1) continue;

			// Ignore guards that we've already warned about
			bool warned = true;
			for (set<item_container>::iterator it = guarded->second.begin(); it != guarded->second.end(); it++) {
				if (warnedGuards.find(*it) == warnedGuards.end()) {
					// Haven't warned about this guard yet, so this is a new kind of clash
					warned = false;
					break;
				}
			}

			if (warned) continue;

			// Default severity is 'warning' for the first item, and 'detail' for the remainder
			bool shownWarning = false;

			// Get the state ID
			const lalr_state_container& state = builder->machine().state_with_id(stateId);

			// Iterate through the guards
			for (set<item_container>::iterator clashingGuard = guarded->second.begin(); clashingGuard != guarded->second.end(); clashingGuard++) {
				// Warned about this guard
				warnedGuards.insert(*clashingGuard);

				// Find all of the LR items that have a reference to this guard
				for (lalr_state::iterator lrItem = state->begin(); lrItem != state->end(); lrItem++) {
					// Ignore at end items
					if ((*lrItem)->at_end()) continue;

					// Ignore items that do not refer to this guard
					// (Assumption is that all usages of the guard have the same initial set, so any symbol refers to any reference to this guard)
					if ((*lrItem)->rule()->items()[(*lrItem)->offset()] != *clashingGuard) continue;

					// Get the position of this rule
					int             ruleId      = (*lrItem)->rule()->identifier(*language->grammar());
					position        rulePos     = language->rule_definition_pos(ruleId);
                    const wstring&  ruleFile    = language->rule_definition_file(ruleId);

					// We know the item, the guard and the position: generate the error message
					if (!shownWarning) {
						wstringstream warning;
						warning	<< L"Found clashing guards on symbol '" 
								<< formatter::to_string(*guarded->first, builder->gram(), builder->terminals())
								<< L"'";
						
						cons.report_error(error(error::sev_warning, ruleFile, L"CLASHING_GUARDS", warning.str(), rulePos));
					}

					// Provide detail messages on where the clash is
					wstringstream msg;
					msg << (!shownWarning?L"when used here: ":L"or here: ")
						<< formatter::to_string(**lrItem, builder->gram(), builder->terminals());
					
					cons.report_error(error(error::sev_detail, language->filename(), L"CLASHING_GUARDS_DETAIL", msg.str(), rulePos));

					shownWarning = true;
				}
			}
		}
	}
}


/// \brief Compiles the parser specified by the parameters to this stage
void lr_parser_stage::compile() {
    // Verbose message to say which stage we're at
	cons().verbose_stream() << L"  = Building parser" << endl;

	// Recycle the parser generator if it already exists
	if (m_Parser) {
		delete m_Parser;
		m_Parser = NULL;
	}
    
    if (m_Tables) {
        delete m_Tables;
        m_Tables = NULL;
    }

	// Sanity check (language)
	if (!m_Language) {
		cons().report_error(error(error::sev_bug, filename(), L"BUG_NO_LANGUAGE", L"Language compiler stage was not supplied to parser stage", m_StartPosition));
		return;
	}

	if (!m_Language->lexer()) {
		cons().report_error(error(error::sev_bug, filename(), L"BUG_NO_LANGUAGE_LEXER", L"Language compiler stage has not generated a lexer", m_StartPosition));
		return;
	}

	if (!m_Language->terminals()) {
		cons().report_error(error(error::sev_bug, filename(), L"BUG_NO_LANGUAGE_TERMINALS", L"Language compiler stage has not generated a terminal dictionary", m_StartPosition));		
{'fraction_non_alphanumeric': 0.12207935849578322, 'fraction_numerical': 0.003041614820959491, 'mean_word_length': 3.9800344234079175, 'pattern_counts': {'":': 2, '<': 55, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6329487', 'n_tokens_mistral': 4910, 'n_tokens_neox': 4329, 'n_words': 1308}