<commit_before>// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "MacroExpander.h"

#include <algorithm>
#include <sstream>

#include "Diagnostics.h"
#include "Token.h"

namespace pp
{

class TokenLexer : public Lexer
{
 public:
	typedef std::vector<Token> TokenVector;

	TokenLexer(TokenVector* tokens)
	{
		tokens->swap(mTokens);
		mIter = mTokens.begin();
	}

	virtual void lex(Token* token)
	{
		if (mIter == mTokens.end())
		{
			token->reset();
			token->type = Token::LAST;
		}
		else
		{
			*token = *mIter++;
		}
	}

 private:
	PP_DISALLOW_COPY_AND_ASSIGN(TokenLexer);

	TokenVector mTokens;
	TokenVector::const_iterator mIter;
};

MacroExpander::MacroExpander(Lexer* lexer,
                             MacroSet* macroSet,
                             Diagnostics* diagnostics,
                             bool parseDefined) :
	mLexer(lexer), mMacroSet(macroSet), mDiagnostics(diagnostics), mParseDefined(parseDefined)
{
	mReserveToken = nullptr;
}

MacroExpander::~MacroExpander()
{
	for (size_t i = 0; i < mContextStack.size(); ++i)
	{
		delete mContextStack[i];
	}

	delete mReserveToken;
}

void MacroExpander::lex(Token* token)
{
	while (true)
	{
		getToken(token);

		if (token->type != Token::IDENTIFIER)
			break;

		// Defined operator is parsed here since it may be generated by macro expansion.
		// Defined operator produced by macro expansion has undefined behavior according to C++
		// spec, which the GLSL spec references (see C++14 draft spec section 16.1.4), but this
		// behavior is needed for passing dEQP tests, which enforce stricter compatibility between
		// implementations.
		if (mParseDefined && token->text == "defined")
		{
			bool paren = false;
			getToken(token);
			if (token->type == '(')
			{
				paren = true;
				getToken(token);
			}
			if (token->type != Token::IDENTIFIER)
			{
				mDiagnostics->report(Diagnostics::UNEXPECTED_TOKEN, token->location,
				                     token->text);
				break;
			}
			auto iter = mMacroSet->find(token->text);
			std::string expression = iter != mMacroSet->end() ? "1" : "0";

			if (paren)
			{
				getToken(token);
				if (token->type != ')')
				{
					mDiagnostics->report(Diagnostics::UNEXPECTED_TOKEN, token->location,
					                     token->text);
					break;
				}
			}

			// We have a valid defined operator.
			// Convert the current token into a CONST_INT token.
			token->type = Token::CONST_INT;
			token->text = expression;
			break;
		}

		if (token->expansionDisabled())
			break;

		MacroSet::const_iterator iter = mMacroSet->find(token->text);
		if (iter == mMacroSet->end())
			break;

		const Macro& macro = iter->second;
		if (macro.disabled)
		{
			// If a particular token is not expanded, it is never expanded.
			token->setExpansionDisabled(true);
			break;
		}
		if ((macro.type == Macro::kTypeFunc) && !isNextTokenLeftParen())
		{
			// If the token immediately after the macro name is not a '(',
			// this macro should not be expanded.
			break;
		}

		pushMacro(macro, *token);
	}
}

void MacroExpander::getToken(Token* token)
{
	if (mReserveToken)
	{
		*token = *mReserveToken;
		delete mReserveToken;
		return;
	}

	// First pop all empty macro contexts.
	while (!mContextStack.empty() && mContextStack.back()->empty())
	{
		popMacro();
	}<commit_msg>Reset pointer to null after delete.<commit_after>// Copyright 2016 The SwiftShader Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "MacroExpander.h"

#include <algorithm>
#include <sstream>

#include "Diagnostics.h"
#include "Token.h"

namespace pp
{

class TokenLexer : public Lexer
{
 public:
	typedef std::vector<Token> TokenVector;

	TokenLexer(TokenVector* tokens)
	{
		tokens->swap(mTokens);
		mIter = mTokens.begin();
	}

	virtual void lex(Token* token)
	{
		if (mIter == mTokens.end())
		{
			token->reset();
			token->type = Token::LAST;
		}
		else
		{
			*token = *mIter++;
		}
	}

 private:
	PP_DISALLOW_COPY_AND_ASSIGN(TokenLexer);

	TokenVector mTokens;
	TokenVector::const_iterator mIter;
};

MacroExpander::MacroExpander(Lexer* lexer,
                             MacroSet* macroSet,
                             Diagnostics* diagnostics,
                             bool parseDefined) :
	mLexer(lexer), mMacroSet(macroSet), mDiagnostics(diagnostics), mParseDefined(parseDefined)
{
	mReserveToken = nullptr;
}

MacroExpander::~MacroExpander()
{
	for (size_t i = 0; i < mContextStack.size(); ++i)
	{
		delete mContextStack[i];
	}

	delete mReserveToken;
}

void MacroExpander::lex(Token* token)
{
	while (true)
	{
		getToken(token);

		if (token->type != Token::IDENTIFIER)
			break;

		// Defined operator is parsed here since it may be generated by macro expansion.
		// Defined operator produced by macro expansion has undefined behavior according to C++
		// spec, which the GLSL spec references (see C++14 draft spec section 16.1.4), but this
		// behavior is needed for passing dEQP tests, which enforce stricter compatibility between
		// implementations.
		if (mParseDefined && token->text == "defined")
		{
			bool paren = false;
			getToken(token);
			if (token->type == '(')
			{
				paren = true;
				getToken(token);
			}
			if (token->type != Token::IDENTIFIER)
			{
				mDiagnostics->report(Diagnostics::UNEXPECTED_TOKEN, token->location,
				                     token->text);
				break;
			}
			auto iter = mMacroSet->find(token->text);
			std::string expression = iter != mMacroSet->end() ? "1" : "0";

			if (paren)
			{
				getToken(token);
				if (token->type != ')')
				{
					mDiagnostics->report(Diagnostics::UNEXPECTED_TOKEN, token->location,
					                     token->text);
					break;
				}
			}

			// We have a valid defined operator.
			// Convert the current token into a CONST_INT token.
			token->type = Token::CONST_INT;
			token->text = expression;
			break;
		}

		if (token->expansionDisabled())
			break;

		MacroSet::const_iterator iter = mMacroSet->find(token->text);
		if (iter == mMacroSet->end())
			break;

		const Macro& macro = iter->second;
		if (macro.disabled)
		{
			// If a particular token is not expanded, it is never expanded.
			token->setExpansionDisabled(true);
			break;
		}
		if ((macro.type == Macro::kTypeFunc) && !isNextTokenLeftParen())
		{
			// If the token immediately after the macro name is not a '(',
			// this macro should not be expanded.
			break;
		}

		pushMacro(macro, *token);
	}
}

void MacroExpander::getToken(Token* token)
{
	if (mReserveToken)
	{
		*token = *mReserveToken;
		delete mReserveToken;
		mReserveToken = nullptr;
		return;
	}

	// First pop all empty macro contexts.
	while (!mContextStack.empty() && mContextStack.back()->empty())
	{
		popMacro();
	}
{'fraction_non_alphanumeric': 0.12421323057161207, 'fraction_numerical': 0.004367373153500321, 'mean_word_length': 3.0978947368421053, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7481303', 'n_tokens_mistral': 2798, 'n_tokens_neox': 2469, 'n_words': 705}