<commit_before>package dyvil.tools.compiler.transform;

import dyvil.tools.parsing.lexer.BaseSymbols;
import dyvil.tools.parsing.lexer.Tokens;
import dyvil.tools.parsing.token.IToken;
import dyvil.tools.parsing.token.InferredSemicolon;

public final class SemicolonInference
{
	private SemicolonInference()
	{
		// no instances
	}
	
	public static void inferSemicolons(IToken first)
	{
		if (first == null)
		{
			return;
		}
		
		IToken prev = first;
		IToken next = first.next();
		while (next.type() != Tokens.EOF)
		{
			inferSemicolon(prev, next);
			prev = next;
			next = next.next();
		}
		
		prev = first;
		next = first.next();
		while (next.type() != Tokens.EOF)
		{
			next.setPrev(prev);
			prev = next;
			next = next.next();
		}
	}
	
	private static void inferSemicolon(IToken prev, IToken next)
	{
		final int prevLine = prev.endLine();
		final int nextLine = next.startLine();
		if (nextLine == prevLine)
		{
			// Don't infer a semicolon
			return;
		}
		
		// Only check tokens if the two tokens are on adjacent lines. Always
		// infer a semicolon if there are one or more blank lines in between
		if (nextLine == prevLine + 1)
		{
			// Check last token on line in question
			final int prevType = prev.type();

			// Check if the previous token is a symbol
			if ((prevType & Tokens.SYMBOL) != 0)
			{
				switch (prevType)
				{
				case DyvilSymbols.UNDERSCORE:
				case DyvilSymbols.ELLIPSIS:
					break; // continue inference checking
				default:
					return; // don't infer a semicolon
				}
			}
			// Check if the previous token is a keyword, but not a value keyword
			else if ((prevType & Tokens.KEYWORD) != 0)
			{
				switch (prevType)
				{
				case DyvilKeywords.TRUE:
				case DyvilKeywords.FALSE:
				case DyvilKeywords.BREAK:
				case DyvilKeywords.CONTINUE:
				case DyvilKeywords.THIS:
				case DyvilKeywords.SUPER:
				case DyvilKeywords.NIL:
				case DyvilKeywords.NULL:
					break; // continue inference checking
				default:
					return; // don't infer a semicolon
				}
			}
			else
			{
				// Check for other token types
				switch (prevType)
				{
				case Tokens.STRING_START:
				case Tokens.STRING_PART:<commit_msg>Hash Symbol to bypass Semicolon Inference<commit_after>package dyvil.tools.compiler.transform;

import dyvil.tools.parsing.lexer.BaseSymbols;
import dyvil.tools.parsing.lexer.Tokens;
import dyvil.tools.parsing.token.IToken;
import dyvil.tools.parsing.token.InferredSemicolon;

public final class SemicolonInference
{
	private SemicolonInference()
	{
		// no instances
	}
	
	public static void inferSemicolons(IToken first)
	{
		if (first == null)
		{
			return;
		}
		
		IToken prev = first;
		IToken next = first.next();
		while (next.type() != Tokens.EOF)
		{
			inferSemicolon(prev, next);
			prev = next;
			next = next.next();
		}
		
		prev = first;
		next = first.next();
		while (next.type() != Tokens.EOF)
		{
			next.setPrev(prev);
			prev = next;
			next = next.next();
		}
	}
	
	private static void inferSemicolon(IToken prev, IToken next)
	{
		final int prevLine = prev.endLine();
		final int nextLine = next.startLine();
		if (nextLine == prevLine)
		{
			// Don't infer a semicolon
			return;
		}
		
		// Only check tokens if the two tokens are on adjacent lines. Always
		// infer a semicolon if there are one or more blank lines in between
		if (nextLine == prevLine + 1)
		{
			// Check last token on line in question
			final int prevType = prev.type();

			// Check if the previous token is a symbol
			if ((prevType & Tokens.SYMBOL) != 0)
			{
				switch (prevType)
				{
				case DyvilSymbols.HASH:
				{
					// Strip the # token
					final IToken prev2 = prev.prev();
					prev2.setNext(next);
					next.setPrev(prev2);
					return; // don't infer a semicolon
				}
				case DyvilSymbols.UNDERSCORE:
				case DyvilSymbols.ELLIPSIS:
					break; // continue inference checking
				default:
					return; // don't infer a semicolon
				}
			}
			// Check if the previous token is a keyword, but not a value keyword
			else if ((prevType & Tokens.KEYWORD) != 0)
			{
				switch (prevType)
				{
				case DyvilKeywords.TRUE:
				case DyvilKeywords.FALSE:
				case DyvilKeywords.BREAK:
				case DyvilKeywords.CONTINUE:
				case DyvilKeywords.RETURN:
				case DyvilKeywords.THIS:
				case DyvilKeywords.SUPER:
				case DyvilKeywords.NIL:
				case DyvilKeywords.NULL:
					break; // continue inference checking
				default:
					return; // don't infer a semicolon
				}
			}
			else
			{
				// Check for other token types
				switch (prevType)
				{
				case Tokens.STRING_START:
				case Tokens.STRING_PART:
{'fraction_non_alphanumeric': 0.10239651416122005, 'fraction_numerical': 0.00196078431372549, 'mean_word_length': 2.732520325203252, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4824250', 'n_tokens_mistral': 1908, 'n_tokens_neox': 1543, 'n_words': 428}