<commit_before>package com.laytonsmith.core;

import com.laytonsmith.abstraction.Implementation;
import com.laytonsmith.annotations.breakable;
import com.laytonsmith.annotations.nolinking;
import com.laytonsmith.annotations.unbreakable;
import com.laytonsmith.commandhelper.CommandHelperPlugin;
import com.laytonsmith.core.Optimizable.OptimizationOption;
import com.laytonsmith.core.compiler.FileOptions;
import com.laytonsmith.core.compiler.KeywordList;
import com.laytonsmith.core.compiler.TokenStream;
import com.laytonsmith.core.constructs.CDecimal;
import com.laytonsmith.core.constructs.CDouble;
import com.laytonsmith.core.constructs.CFunction;
import com.laytonsmith.core.constructs.CIdentifier;
import com.laytonsmith.core.constructs.CInt;
import com.laytonsmith.core.constructs.CKeyword;
import com.laytonsmith.core.constructs.CLabel;
import com.laytonsmith.core.constructs.CNull;
import com.laytonsmith.core.constructs.CPreIdentifier;
import com.laytonsmith.core.constructs.CSlice;
import com.laytonsmith.core.constructs.CString;
import com.laytonsmith.core.constructs.CSymbol;
import com.laytonsmith.core.constructs.CVoid;
import com.laytonsmith.core.constructs.Construct;
import com.laytonsmith.core.constructs.IVariable;
import com.laytonsmith.core.constructs.Target;
import com.laytonsmith.core.constructs.Token;
import com.laytonsmith.core.constructs.Token.TType;
import com.laytonsmith.core.constructs.Variable;
import com.laytonsmith.core.environments.CommandHelperEnvironment;
import com.laytonsmith.core.environments.Environment;
import com.laytonsmith.core.environments.GlobalEnv;
import com.laytonsmith.core.exceptions.ConfigCompileException;
import com.laytonsmith.core.exceptions.ConfigCompileGroupException;
import com.laytonsmith.core.exceptions.ConfigRuntimeException;
import com.laytonsmith.core.exceptions.ProgramFlowManipulationException;
import com.laytonsmith.core.functions.Compiler;
import com.laytonsmith.core.functions.DataHandling;
import com.laytonsmith.core.functions.Function;
import com.laytonsmith.core.functions.FunctionBase;
import com.laytonsmith.core.functions.FunctionList;
import com.laytonsmith.core.functions.IncludeCache;
import com.laytonsmith.core.taskmanager.TaskManager;
import com.laytonsmith.persistence.DataSourceException;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EmptyStackException;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Stack;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;

/**
 * The MethodScriptCompiler class handles the various stages of compilation and provides helper methods for execution of
 * the compiled trees.
 */
public final class MethodScriptCompiler {

    private final static EnumSet<Optimizable.OptimizationOption> NO_OPTIMIZATIONS = EnumSet.noneOf(Optimizable.OptimizationOption.class);

    private MethodScriptCompiler() {
    }

    private static final Pattern VAR_PATTERN = Pattern.compile("\\$[\\p{L}0-9_]+");
    private static final Pattern IVAR_PATTERN = Pattern.compile(IVariable.VARIABLE_NAME_REGEX);

    /**
     * Lexes the script, and turns it into a token stream. This looks through the script character by character.
     *
     * @param script The script to lex
     * @param file The file this script came from, or potentially null if the code is from a dynamic source
     * @param inPureMScript If the script is in pure MethodScript, this should be true. Pure MethodScript is defined as
     * code that doesn't have command alias wrappers.
     * @return A stream of tokens
     * @throws ConfigCompileException If compilation fails due to bad syntax
     */
    public static TokenStream lex(String script, File file, boolean inPureMScript) throws ConfigCompileException {
        return lex(script, file, inPureMScript, false);
    }

	/**
	 * Lexes the script, and turns it into a token stream. This looks through the script character by character.
	 *
	 * @param script The script to lex
	 * @param file The file this script came from, or potentially null if the code is from a dynamic source
	 * @param inPureMScript If the script is in pure MethodScript, this should be true. Pure MethodScript is defined as
	 * code that doesn't have command alias wrappers.
	 * @param saveAllTokens If this script is planning to be compiled, then this value should always be false, however,
	 * if the calling code needs all tokens for informational purposes (and doesn't plan on actually compiling the code)
	 * then this can be true. If true, all tokens are saved, including comments and (some) whitespace. Given this lexing
	 * stream, the exact source code could be re-constructed.
	 *
	 * A note on whitespace: The whitespace tokens are not guaranteed to be accurate, however, the column information
	 * is. If you have two tokens t1 and t2, each with a value of length 1, where the columns are 1 and 5, then that
	 * means there are 4 spaces between the two.
	 * @return A stream of tokens
	 * @throws ConfigCompileException If compilation fails due to bad syntax
	 */
	public static TokenStream lex(String script, File file, boolean inPureMScript, boolean saveAllTokens) throws ConfigCompileException {
		if(script.isEmpty()) {
			return new TokenStream(new ArrayList<>(), "");
		}
		if((int) script.charAt(0) == 65279) {
			// Remove the UTF-8 Byte Order Mark, if present.
			script = script.substring(1);
		}
		StringBuilder fileOptions = new StringBuilder();
		StringBuilder fileOptionsBuf = null;
		script = script.replaceAll("\r\n", "\n");
		script = script + "\n";
		Set<String> keywords = KeywordList.getKeywordNames();
		List<Token> token_list = new ArrayList<>();
		
		// Set our state variables.
		boolean state_in_quote = false;
		int quoteLineNumberStart = 1;
		boolean in_smart_quote = false;
		int smartQuoteLineNumberStart = 1;
		boolean in_comment = false;
		int commentLineNumberStart = 1;
		boolean comment_is_block = false;
		boolean in_opt_var = false;
		boolean inCommand = (!inPureMScript);
		boolean inMultiline = false;
		boolean in_hash_comment = false;
		boolean in_smart_comment = false;
		boolean in_file_options = false;
		int fileOptionsLineNumberStart = 1;
		
		StringBuilder buf = new StringBuilder();
		int line_num = 1;
		int column = 1;
		int lastColumn = 0;
		Target target = Target.UNKNOWN;
		
		// Lex the script character by character.
		for(int i = 0; i < script.length(); i++) {
			Character c = script.charAt(i);
			Character c2 = null;
			if(i < script.length() - 1) {
				c2 = script.charAt(i + 1);
			}
			
			column += i - lastColumn;
			lastColumn = i;
			if(c == '\n') {
				line_num++;
				column = 1;
				if(!inMultiline && !inPureMScript) {
					inCommand = true;
				}
			}
			if(buf.length() == 0) {
				target = new Target(line_num, file, column);
			}
			
			// Comment handling. This is bypassed if we are in a string.
			if(!state_in_quote && !in_smart_quote) {
				switch(c) {
					
					// Block comments start (/* and /**) and Double slash line comment start (//).
					case '/': {
						if(!in_comment) {
							if(c2 == '*') { // "/*" or "/**".
								buf.append("/*");
								in_comment = true;
								comment_is_block = true;
								if(i < script.length() - 2 && script.charAt(i + 2) == '*') { // "/**".
									in_smart_comment = true;
									buf.append("*");
									i++;
								}
								commentLineNumberStart = line_num;
								i++;
								
								// Handle file option tokens.
								if(saveAllTokens && fileOptionsBuf != null && fileOptionsBuf.length() > 0) {
									token_list.add(new Token(
											TType.FILE_OPTIONS_STRING, fileOptionsBuf.toString(), target));
									fileOptions.append(fileOptionsBuf);
									fileOptionsBuf = new StringBuilder();
								}
								
								continue;
							} else if(c2 == '/') { // "//".
								buf.append("//");
								in_comment = true;
								comment_is_block = false;
								i++;
								
								// Handle file option tokens.
								if(saveAllTokens && fileOptionsBuf != null && fileOptionsBuf.length() > 0) {
									token_list.add(new Token(
											TType.FILE_OPTIONS_STRING, fileOptionsBuf.toString(), target));
									fileOptions.append(fileOptionsBuf);
									fileOptionsBuf = new StringBuilder();
								}
								
								continue;
							}
						}
						break;
					}
					
					// Line comment start (#).
					case '#': {
						if(!in_comment) { // "#".
							buf.append("#");
							in_comment = true;
							comment_is_block = false;
							in_hash_comment = true;
							
							// Handle file option tokens.
							if(saveAllTokens && fileOptionsBuf != null && fileOptionsBuf.length() > 0) {
								token_list.add(new Token(
										TType.FILE_OPTIONS_STRING, fileOptionsBuf.toString(), target));
								fileOptions.append(fileOptionsBuf);
								fileOptionsBuf = new StringBuilder();
							}
							
							continue;
						}
						break;
					}
					
					// Block comment end (*/).
					case '*': {
						if(in_comment && comment_is_block && c2 == '/') { // "*/".
							if(saveAllTokens || in_smart_comment) {
								buf.append("*/");
								token_list.add(new Token(in_smart_comment ? TType.SMART_COMMENT : TType.COMMENT,
										buf.toString(), target));
							}
							buf = new StringBuilder();
							target = new Target(line_num, file, column);
							in_comment = false;
							comment_is_block = false;
							in_smart_comment = false;
							i++;
							continue;
						}
						break;
					}
					
					// Line comment end (\n).
					case '\n': {
						if(in_comment && !comment_is_block) { // "\n".
							in_comment = false;
							in_hash_comment = false;
							if(saveAllTokens) {
								token_list.add(new Token(TType.COMMENT, buf.toString(), target));
								token_list.add(new Token(TType.NEWLINE, "\n", new Target(line_num + 1, file, 0)));
							}
							buf = new StringBuilder();
							target = new Target(line_num, file, column);
							continue;
						}
						break;
					}
				}
			}
			
			// If we are in a comment, add the character to the buffer.
			if(in_comment) {
				buf.append(c);
				continue;
			}
			
			// If we are in file options, add the character to the buffer if it's not a file options end character.
			if(in_file_options) {
				// For a '>' character outside of a comment, '\>' would have to be used in file options.
				// Other characters than '>'cannot be escaped.
				// If support for more escaped characters would be desired in the future, it could be added here.
				switch(c) {<commit_msg>Removed unnecessary assigns in the lexer.<commit_after>package com.laytonsmith.core;

import com.laytonsmith.abstraction.Implementation;
import com.laytonsmith.annotations.breakable;
import com.laytonsmith.annotations.nolinking;
import com.laytonsmith.annotations.unbreakable;
import com.laytonsmith.commandhelper.CommandHelperPlugin;
import com.laytonsmith.core.Optimizable.OptimizationOption;
import com.laytonsmith.core.compiler.FileOptions;
import com.laytonsmith.core.compiler.KeywordList;
import com.laytonsmith.core.compiler.TokenStream;
import com.laytonsmith.core.constructs.CDecimal;
import com.laytonsmith.core.constructs.CDouble;
import com.laytonsmith.core.constructs.CFunction;
import com.laytonsmith.core.constructs.CIdentifier;
import com.laytonsmith.core.constructs.CInt;
import com.laytonsmith.core.constructs.CKeyword;
import com.laytonsmith.core.constructs.CLabel;
import com.laytonsmith.core.constructs.CNull;
import com.laytonsmith.core.constructs.CPreIdentifier;
import com.laytonsmith.core.constructs.CSlice;
import com.laytonsmith.core.constructs.CString;
import com.laytonsmith.core.constructs.CSymbol;
import com.laytonsmith.core.constructs.CVoid;
import com.laytonsmith.core.constructs.Construct;
import com.laytonsmith.core.constructs.IVariable;
import com.laytonsmith.core.constructs.Target;
import com.laytonsmith.core.constructs.Token;
import com.laytonsmith.core.constructs.Token.TType;
import com.laytonsmith.core.constructs.Variable;
import com.laytonsmith.core.environments.CommandHelperEnvironment;
import com.laytonsmith.core.environments.Environment;
import com.laytonsmith.core.environments.GlobalEnv;
import com.laytonsmith.core.exceptions.ConfigCompileException;
import com.laytonsmith.core.exceptions.ConfigCompileGroupException;
import com.laytonsmith.core.exceptions.ConfigRuntimeException;
import com.laytonsmith.core.exceptions.ProgramFlowManipulationException;
import com.laytonsmith.core.functions.Compiler;
import com.laytonsmith.core.functions.DataHandling;
import com.laytonsmith.core.functions.Function;
import com.laytonsmith.core.functions.FunctionBase;
import com.laytonsmith.core.functions.FunctionList;
import com.laytonsmith.core.functions.IncludeCache;
import com.laytonsmith.core.taskmanager.TaskManager;
import com.laytonsmith.persistence.DataSourceException;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EmptyStackException;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Stack;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;

/**
 * The MethodScriptCompiler class handles the various stages of compilation and provides helper methods for execution of
 * the compiled trees.
 */
public final class MethodScriptCompiler {

    private final static EnumSet<Optimizable.OptimizationOption> NO_OPTIMIZATIONS = EnumSet.noneOf(Optimizable.OptimizationOption.class);

    private MethodScriptCompiler() {
    }

    private static final Pattern VAR_PATTERN = Pattern.compile("\\$[\\p{L}0-9_]+");
    private static final Pattern IVAR_PATTERN = Pattern.compile(IVariable.VARIABLE_NAME_REGEX);

    /**
     * Lexes the script, and turns it into a token stream. This looks through the script character by character.
     *
     * @param script The script to lex
     * @param file The file this script came from, or potentially null if the code is from a dynamic source
     * @param inPureMScript If the script is in pure MethodScript, this should be true. Pure MethodScript is defined as
     * code that doesn't have command alias wrappers.
     * @return A stream of tokens
     * @throws ConfigCompileException If compilation fails due to bad syntax
     */
    public static TokenStream lex(String script, File file, boolean inPureMScript) throws ConfigCompileException {
        return lex(script, file, inPureMScript, false);
    }

	/**
	 * Lexes the script, and turns it into a token stream. This looks through the script character by character.
	 *
	 * @param script The script to lex
	 * @param file The file this script came from, or potentially null if the code is from a dynamic source
	 * @param inPureMScript If the script is in pure MethodScript, this should be true. Pure MethodScript is defined as
	 * code that doesn't have command alias wrappers.
	 * @param saveAllTokens If this script is planning to be compiled, then this value should always be false, however,
	 * if the calling code needs all tokens for informational purposes (and doesn't plan on actually compiling the code)
	 * then this can be true. If true, all tokens are saved, including comments and (some) whitespace. Given this lexing
	 * stream, the exact source code could be re-constructed.
	 *
	 * A note on whitespace: The whitespace tokens are not guaranteed to be accurate, however, the column information
	 * is. If you have two tokens t1 and t2, each with a value of length 1, where the columns are 1 and 5, then that
	 * means there are 4 spaces between the two.
	 * @return A stream of tokens
	 * @throws ConfigCompileException If compilation fails due to bad syntax
	 */
	public static TokenStream lex(String script, File file, boolean inPureMScript, boolean saveAllTokens) throws ConfigCompileException {
		if(script.isEmpty()) {
			return new TokenStream(new ArrayList<>(), "");
		}
		if((int) script.charAt(0) == 65279) {
			// Remove the UTF-8 Byte Order Mark, if present.
			script = script.substring(1);
		}
		final StringBuilder fileOptions = new StringBuilder();
		StringBuilder fileOptionsBuf = null;
		script = script.replaceAll("\r\n", "\n");
		script = script + "\n";
		final Set<String> keywords = KeywordList.getKeywordNames();
		final List<Token> token_list = new ArrayList<>();
		
		// Set our state variables.
		boolean state_in_quote = false;
		int quoteLineNumberStart = 1;
		boolean in_smart_quote = false;
		int smartQuoteLineNumberStart = 1;
		boolean in_comment = false;
		int commentLineNumberStart = 1;
		boolean comment_is_block = false;
		boolean in_opt_var = false;
		boolean inCommand = (!inPureMScript);
		boolean inMultiline = false;
		boolean in_smart_comment = false;
		boolean in_file_options = false;
		int fileOptionsLineNumberStart = 1;
		
		StringBuilder buf = new StringBuilder();
		int line_num = 1;
		int column = 1;
		int lastColumn = 0;
		Target target = Target.UNKNOWN;
		
		// Lex the script character by character.
		for(int i = 0; i < script.length(); i++) {
			Character c = script.charAt(i);
			Character c2 = null;
			if(i < script.length() - 1) {
				c2 = script.charAt(i + 1);
			}
			
			column += i - lastColumn;
			lastColumn = i;
			if(c == '\n') {
				line_num++;
				column = 1;
				if(!inMultiline && !inPureMScript) {
					inCommand = true;
				}
			}
			if(buf.length() == 0) {
				target = new Target(line_num, file, column);
			}
			
			// Comment handling. This is bypassed if we are in a string.
			if(!state_in_quote && !in_smart_quote) {
				switch(c) {
					
					// Block comments start (/* and /**) and Double slash line comment start (//).
					case '/': {
						if(!in_comment) {
							if(c2 == '*') { // "/*" or "/**".
								buf.append("/*");
								in_comment = true;
								comment_is_block = true;
								if(i < script.length() - 2 && script.charAt(i + 2) == '*') { // "/**".
									in_smart_comment = true;
									buf.append("*");
									i++;
								}
								commentLineNumberStart = line_num;
								i++;
								
								// Handle file option tokens.
								if(saveAllTokens && fileOptionsBuf != null && fileOptionsBuf.length() > 0) {
									token_list.add(new Token(
											TType.FILE_OPTIONS_STRING, fileOptionsBuf.toString(), target));
									fileOptions.append(fileOptionsBuf);
									fileOptionsBuf = new StringBuilder();
								}
								
								continue;
							} else if(c2 == '/') { // "//".
								buf.append("//");
								in_comment = true;
								i++;
								
								// Handle file option tokens.
								if(saveAllTokens && fileOptionsBuf != null && fileOptionsBuf.length() > 0) {
									token_list.add(new Token(
											TType.FILE_OPTIONS_STRING, fileOptionsBuf.toString(), target));
									fileOptions.append(fileOptionsBuf);
									fileOptionsBuf = new StringBuilder();
								}
								
								continue;
							}
						}
						break;
					}
					
					// Line comment start (#).
					case '#': {
						if(!in_comment) { // "#".
							buf.append("#");
							in_comment = true;
							
							// Handle file option tokens.
							if(saveAllTokens && fileOptionsBuf != null && fileOptionsBuf.length() > 0) {
								token_list.add(new Token(
										TType.FILE_OPTIONS_STRING, fileOptionsBuf.toString(), target));
								fileOptions.append(fileOptionsBuf);
								fileOptionsBuf = new StringBuilder();
							}
							
							continue;
						}
						break;
					}
					
					// Block comment end (*/).
					case '*': {
						if(in_comment && comment_is_block && c2 == '/') { // "*/".
							if(saveAllTokens || in_smart_comment) {
								buf.append("*/");
								token_list.add(new Token(in_smart_comment ? TType.SMART_COMMENT : TType.COMMENT,
										buf.toString(), target));
							}
							buf = new StringBuilder();
							target = new Target(line_num, file, column);
							in_comment = false;
							comment_is_block = false;
							in_smart_comment = false;
							i++;
							continue;
						}
						break;
					}
					
					// Line comment end (\n).
					case '\n': {
						if(in_comment && !comment_is_block) { // "\n".
							in_comment = false;
							if(saveAllTokens) {
								token_list.add(new Token(TType.COMMENT, buf.toString(), target));
								token_list.add(new Token(TType.NEWLINE, "\n", new Target(line_num + 1, file, 0)));
							}
							buf = new StringBuilder();
							target = new Target(line_num, file, column);
							continue;
						}
						break;
					}
				}
			}
			
			// If we are in a comment, add the character to the buffer.
			if(in_comment) {
				buf.append(c);
				continue;
			}
			
			// If we are in file options, add the character to the buffer if it's not a file options end character.
			if(in_file_options) {
				// For a '>' character outside of a comment, '\>' would have to be used in file options.
				// Other characters than '>'cannot be escaped.
				// If support for more escaped characters would be desired in the future, it could be added here.
				switch(c) {
{'fraction_non_alphanumeric': 0.1051815662141213, 'fraction_numerical': 0.0037581622586555174, 'mean_word_length': 3.189726431804763, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6814532', 'n_tokens_mistral': 7903, 'n_tokens_neox': 6158, 'n_words': 1759}