<commit_before>/**
 * Zillians MMO
 * Copyright (C) 2007-2011 Zillians.com, Inc.
 * For more information see http://www.zillians.com
 *
 * Zillians MMO is the library and runtime for massive multiplayer online game
 * development in utility computing model, which runs as a service for every
 * developer to build their virtual world running on our GPU-assisted machines.
 *
 * This is a close source library intended to be used solely within Zillians.com
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
 * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#include "language/stage/parser/ThorScriptParserStage.h"
#include "language/context/ParserContext.h"
#include "language/grammar/ThorScript.h"
#include "language/action/SemanticActions.h"
#include "language/tree/visitor/PrettyPrintVisitor.h"
#include "language/ThorScriptCompiler.h"
#include "utility/Foreach.h"
#include "utility/UnicodeUtil.h"
#include <boost/filesystem.hpp>

namespace classic = boost::spirit::classic;
namespace qi = boost::spirit::qi;

namespace zillians { namespace language { namespace stage {


namespace {

// since '\t' may be printed in spaces and I don't know a way to change std::wcout, we simply replace the tab with desired number of spaces
// so that we can have correct error pointing cursor
// (note that because '\t' equals to 4 spaces reported by spirit, we have to make sure it's printed in the exact same way)
static void expand_tabs(const std::wstring& input, std::wstring& output, int number_of_space_for_tab = 4)
{
	for(std::wstring::const_iterator it = input.begin(); it != input.end(); ++it)
	{
		if(*it == '\t')
			for(int i=0;i<number_of_space_for_tab;++i) output.push_back(L' ');
		else
			output.push_back(*it);
	}
}

}

ThorScriptParserStage::ThorScriptParserStage() : debug_parser(false), debug_ast(false), debug_ast_with_loc(false), use_relative_path(false)
{ }

ThorScriptParserStage::~ThorScriptParserStage()
{ }

const char* ThorScriptParserStage::name()
{
	return "Parser Stage";
}

std::pair<shared_ptr<po::options_description>, shared_ptr<po::options_description>> ThorScriptParserStage::getOptions()
{
	shared_ptr<po::options_description> option_desc_public(new po::options_description());
	shared_ptr<po::options_description> option_desc_private(new po::options_description());

	foreach(i, option_desc_public->options()) option_desc_private->add(*i);

	option_desc_private->add_options()
		("debug-parser", "dump parsing tree for debugging purpose")
		("debug-parser-ast", "dump parsed abstract syntax tree for debugging purpose")
		("debug-parser-ast-with-loc", "dump parsed abstract syntax tree with location for debugging purpose")
		("use-relative-path", "use relative file path instead of absolute path (for debugging info generation)");

	return std::make_pair(option_desc_public, option_desc_private);
}

bool ThorScriptParserStage::parseOptions(po::variables_map& vm)
{
	debug_parser = (vm.count("debug-parser") > 0);
	debug_ast = (vm.count("debug-parser-ast") > 0);
	debug_ast_with_loc = (vm.count("debug-parser-ast-with-loc") > 0);
	use_relative_path = (vm.count("use-relative-path") > 0);

	if(vm.count("input") == 0)
		return false;

	inputs = vm["input"].as<std::vector<std::string>>();
	return true;
}

bool ThorScriptParserStage::execute(bool& continue_execution)
{
	// prepare the global parser context
	if(!hasParserContext())
		setParserContext(new ParserContext());

	// prepare the module source info context for the root program node
   	ModuleSourceInfoContext::set(getParserContext().program, new ModuleSourceInfoContext());

	if(inputs.size() > 0)
		foreach(i, inputs)
			if(!parse(*i))
				return false;
	return true;
}

bool ThorScriptParserStage::parse(std::string filename)
{
	std::ifstream in(filename, std::ios_base::in);

	if(!in.good())
	{
		LOG4CXX_ERROR(LoggerWrapper::ParserStage, "failed to open file: " << filename);
		return false;
	}

    // ignore the BOM marking the beginning of a UTF-8 file in Windows
    if(in.peek() == '\xef')
    {
        char s[3];
        in >> s[0] >> s[1] >> s[2];
        s[3] = '\0';<commit_msg>parse only input with .t extension<commit_after>/**
 * Zillians MMO
 * Copyright (C) 2007-2011 Zillians.com, Inc.
 * For more information see http://www.zillians.com
 *
 * Zillians MMO is the library and runtime for massive multiplayer online game
 * development in utility computing model, which runs as a service for every
 * developer to build their virtual world running on our GPU-assisted machines.
 *
 * This is a close source library intended to be used solely within Zillians.com
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
 * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#include "language/stage/parser/ThorScriptParserStage.h"
#include "language/context/ParserContext.h"
#include "language/grammar/ThorScript.h"
#include "language/action/SemanticActions.h"
#include "language/tree/visitor/PrettyPrintVisitor.h"
#include "language/ThorScriptCompiler.h"
#include "utility/Foreach.h"
#include "utility/UnicodeUtil.h"
#include <boost/filesystem.hpp>

namespace classic = boost::spirit::classic;
namespace qi = boost::spirit::qi;

namespace zillians { namespace language { namespace stage {


namespace {

// since '\t' may be printed in spaces and I don't know a way to change std::wcout, we simply replace the tab with desired number of spaces
// so that we can have correct error pointing cursor
// (note that because '\t' equals to 4 spaces reported by spirit, we have to make sure it's printed in the exact same way)
static void expand_tabs(const std::wstring& input, std::wstring& output, int number_of_space_for_tab = 4)
{
	for(std::wstring::const_iterator it = input.begin(); it != input.end(); ++it)
	{
		if(*it == '\t')
			for(int i=0;i<number_of_space_for_tab;++i) output.push_back(L' ');
		else
			output.push_back(*it);
	}
}

}

ThorScriptParserStage::ThorScriptParserStage() : debug_parser(false), debug_ast(false), debug_ast_with_loc(false), use_relative_path(false)
{ }

ThorScriptParserStage::~ThorScriptParserStage()
{ }

const char* ThorScriptParserStage::name()
{
	return "Parser Stage";
}

std::pair<shared_ptr<po::options_description>, shared_ptr<po::options_description>> ThorScriptParserStage::getOptions()
{
	shared_ptr<po::options_description> option_desc_public(new po::options_description());
	shared_ptr<po::options_description> option_desc_private(new po::options_description());

	foreach(i, option_desc_public->options()) option_desc_private->add(*i);

	option_desc_private->add_options()
		("debug-parser", "dump parsing tree for debugging purpose")
		("debug-parser-ast", "dump parsed abstract syntax tree for debugging purpose")
		("debug-parser-ast-with-loc", "dump parsed abstract syntax tree with location for debugging purpose")
		("use-relative-path", "use relative file path instead of absolute path (for debugging info generation)");

	return std::make_pair(option_desc_public, option_desc_private);
}

bool ThorScriptParserStage::parseOptions(po::variables_map& vm)
{
	debug_parser = (vm.count("debug-parser") > 0);
	debug_ast = (vm.count("debug-parser-ast") > 0);
	debug_ast_with_loc = (vm.count("debug-parser-ast-with-loc") > 0);
	use_relative_path = (vm.count("use-relative-path") > 0);

	if(vm.count("input") == 0)
		return false;

	inputs = vm["input"].as<std::vector<std::string>>();
	return true;
}

bool ThorScriptParserStage::execute(bool& continue_execution)
{
	// prepare the global parser context
	if(!hasParserContext())
		setParserContext(new ParserContext());

	// prepare the module source info context for the root program node
   	ModuleSourceInfoContext::set(getParserContext().program, new ModuleSourceInfoContext());

	foreach(i, inputs)
	{
		boost::filesystem::path p(*i);
		if(strcmp(p.extension().c_str(), ".t") == 0)
			if(!parse(*i))
				return false;
	}

	return true;
}

bool ThorScriptParserStage::parse(std::string filename)
{
	std::ifstream in(filename, std::ios_base::in);

	if(!in.good())
	{
		LOG4CXX_ERROR(LoggerWrapper::ParserStage, "failed to open file: " << filename);
		return false;
	}

    // ignore the BOM marking the beginning of a UTF-8 file in Windows
    if(in.peek() == '\xef')
    {
        char s[3];
        in >> s[0] >> s[1] >> s[2];
        s[3] = '\0';
{'fraction_non_alphanumeric': 0.125, 'fraction_numerical': 0.005555555555555556, 'mean_word_length': 5.22045611610228, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '262727', 'n_tokens_mistral': 2866, 'n_tokens_neox': 2728, 'n_words': 825}