<commit_before>/******************************************************************************\
  Expression 2 Parser for Garry's Mod
  <NAME>" Svensson, <EMAIL>
\******************************************************************************/

AddCSLuaFile("parser.lua")

/*

Root
 1 : q1

seQuencing
 1 : ""
 2 : "s1 q1", "s1, q2"

Statement
 1 : if (e1) { q1 } i1
 2 : while (e1) { q1 }
 3 : for (var = e1, e1[, e1]) { q1 }
 4 : foreach(var, var:type = e1) { q1}
 5 : break, continue
 6 : var++, var--
 7 : var += e1, var -= e1, var *= e1, var /= e1
 8 : var = s7, var[e1,type] = s7
 9 : e1

If
 1 : elseif (e1) { q1 } i1
 2 : else { q1 }

Expression
 1 : var = e1, var += e1, var -= e1, var *= e1, var /= e1 [ERROR]
 2 : e3 ? e1 : e1, e3 ?: e1
 3 : e3 | e4
 4 : e4 & e5
 5 : e5 == e6, e5 != e6
 6 : e6 < e7, e6 > e7, e6 <= e7, e6 >= e7
 7 : e7 + e8, e7 - e8
 8 : e8 * e9, e8 / e9, e8 % e9
 9 : e9 ^ e10
10 : +e11, -e11, !e10
11 : e11:fun([e1, ...]), e11[var,type]
12 : (e1), fun([e1, ...])
13 : string, num, ~var, $var, ->var
14 : var++, var-- [ERROR]
15 : var

*/
/******************************************************************************/

Parser = {}
Parser.__index = Parser

function Parser.Execute(...)
	-- instantiate Parser
	local instance = setmetatable({}, Parser)

	-- and pcall the new instance's Process method.
	return pcall(Parser.Process, instance, ...)
end

function Parser:Error(message, token)
	if token then
		error(message .. " at line " .. token[4] .. ", char " .. token[5], 0)
	else
		error(message .. " at line " .. self.token[4] .. ", char " .. self.token[5], 0)
	end
end

function Parser:Process(tokens, params)
	self.tokens = tokens

	self.index = 0
	self.count = #tokens
	self.delta = {}

	self:NextToken()
	local tree = self:Root()

	return tree, self.delta
end

/******************************************************************************/

function Parser:GetToken()
	return self.token
end

function Parser:GetTokenData()
	return self.token[2]
end

function Parser:GetTokenTrace()
	return {self.token[4], self.token[5]}
end


function Parser:Instruction(trace, name, ...)
	return {name, trace, ...} //
end


function Parser:HasTokens()
	return self.readtoken != nil
end

function Parser:NextToken()
	if self.index <= self.count then
		if self.index > 0 then
			self.token = self.readtoken
		else
			self.token = {"", "", false, 1, 1}
		end

		self.index = self.index + 1
		self.readtoken = self.tokens[self.index]
	else
		self.readtoken = nil
	end
end

function Parser:TrackBack()
	self.index = self.index - 2
	self:NextToken()
end


function Parser:AcceptRoamingToken(name)
	local token = self.readtoken
	if !token or token[1] != name then return false end

	self:NextToken()
	return true
end

function Parser:AcceptTailingToken(name)
	local token = self.readtoken
	if !token or token[3] then return false end

	return self:AcceptRoamingToken(name)
end

function Parser:AcceptLeadingToken(name)
	local token = self.tokens[self.index + 1]
	if !token or token[3] then return false end

	return self:AcceptRoamingToken(name)
end


function Parser:RecurseLeft(func, tbl)
	local expr = func(self)
	local hit = true

	while hit do
		hit = false
		for i=1,#tbl do
			if self:AcceptRoamingToken(tbl[i]) then
				local trace = self:GetTokenTrace()

				hit = true
				expr = self:Instruction(trace, tbl[i], expr, func(self))
				break
			end
		end
	end

	return expr
end

/******************************************************************************/

local loopdepth

function Parser:Root()
	loopdepth = 0
	return self:Stmts()
end


function Parser:Stmts()
	local trace = self:GetTokenTrace()
	local stmts = self:Instruction(trace, "seq")

	if !self:HasTokens() then return stmts end

	while true do
		if self:AcceptRoamingToken("com") then
			self:Error("Statement separator (,) must not appear multiple times")
		end

		stmts[#stmts + 1] = self:Stmt1()

		if !self:HasTokens() then break end

		if !self:AcceptRoamingToken("com") then
			if self.readtoken[3] == false then
				self:Error("Statements must be separated by comma (,) or whitespace")
			end
		end
	end

	return stmts
end


function Parser:Stmt1()
	if self:AcceptRoamingToken("if") then
		local trace = self:GetTokenTrace()
		return self:Instruction(trace, "if", self:Cond(), self:Block("if condition"), self:IfElseIf())
	end

	return self:Stmt2()
end

function Parser:Stmt2()
	if self:AcceptRoamingToken("whl") then
		local trace = self:GetTokenTrace()
		loopdepth = loopdepth + 1
		local whl = self:Instruction(trace, "whl", self:Cond(), self:Block("while condition"))
		loopdepth = loopdepth - 1
		return whl
	end

	return self:Stmt3()
end

function Parser:Stmt3()
	if self:AcceptRoamingToken("for") then
		local trace = self:GetTokenTrace()
		loopdepth = loopdepth + 1

		if !self:AcceptRoamingToken("lpa") then
			self:Error("Left parenthesis (() must appear before condition")
		end

		if !self:AcceptRoamingToken("var") then
			self:Error("Variable expected for the numeric index")
		end

		local var = self:GetTokenData()

		if !self:AcceptRoamingToken("ass") then
			self:Error("Assignment operator (=) expected to preceed variable")
		end

		local estart = self:Expr1()

		if !self:AcceptRoamingToken("com") then
			self:Error("Comma (,) expected after start value")
		end

		local estop = self:Expr1()

		local estep
		if self:AcceptRoamingToken("com") then
			estep = self:Expr1()
		end

		if !self:AcceptRoamingToken("rpa") then
			self:Error("Right parenthesis ()) missing, to close condition")
		end

		local sfor = self:Instruction(trace, "for", var, estart, estop, estep, self:Block("for statement"))

		loopdepth = loopdepth - 1
		return sfor
	end

	return self:Stmt4()
end

function Parser:Stmt4()
	if self:AcceptRoamingToken("fea") then
		local trace = self:GetTokenTrace()
		loopdepth = loopdepth + 1

		if not self:AcceptRoamingToken("lpa") then
			self:Error("Left parenthesis missing (() after foreach statement")
		end

		if not self:AcceptRoamingToken("var") then
			self:Error("Variable expected to hold the key")
		end
		local keyvar = self:GetTokenData()

		if not self:AcceptRoamingToken("com") then
			self:Error("Comma (,) expected after key variable")
		end

		if not self:AcceptRoamingToken("var") then
			self:Error("Variable expected to hold the value")
		end
		local valvar = self:GetTokenData()

		if not self:AcceptRoamingToken("col") then
			self:Error("Colon (:) expected to separate type from variable")
		end

		if not self:AcceptRoamingToken("fun") and not self:AcceptRoamingToken("udf") then
			self:Error("Type expected after colon")
		end
		local valtype = self:GetTokenData()
		if valtype == "number" then valtype = "normal" end
		if wire_expression_types[string.upper(valtype)] == nil then
			self:Error("Unknown type: "..valtype)
		end
		valtype = wire_expression_types[string.upper(valtype)][1]

		if not self:AcceptRoamingToken("ass") then
			self:Error("Equals sign (=) expected after value type to specify table")
		end

		local tableexpr = self:Expr1()

		if not self:AcceptRoamingToken("rpa") then
			self:Error("Missing right parenthesis after foreach statement")
		end

		local sfea = self:Instruction(trace, "fea", keyvar, valvar, valtype, tableexpr, self:Block("foreach statement"))
		loopdepth = loopdepth -1
		return sfea
	end

	return self:Stmt5()
end

function Parser:Stmt5()
	if self:AcceptRoamingToken("brk") then
		if loopdepth > 0 then
			local trace = self:GetTokenTrace()
			return self:Instruction(trace, "brk")
		else
			self:Error("Break may not exist outside of a loop")
		end
	elseif self:AcceptRoamingToken("cnt") then
		if loopdepth > 0 then
			local trace = self:GetTokenTrace()
			return self:Instruction(trace, "cnt")
		else
			self:Error("Continue may not exist outside of a loop")
		end
	end

	return self:Stmt6()
end

function Parser:Stmt6()
	if self:AcceptRoamingToken("var") then
		local trace = self:GetTokenTrace()
		local var = self:GetTokenData()

		if self:AcceptTailingToken("inc") then
			return self:Instruction(trace, "inc", var)
		elseif self:AcceptRoamingToken("inc") then
			self:Error("Increment operator (++) must not be preceded by whitespace")
		end

		if self:AcceptTailingToken("dec") then
			return self:Instruction(trace, "dec", var)
		elseif self:AcceptRoamingToken("dec") then
			self:Error("Decrement operator (--) must not be preceded by whitespace")
		end

		self:TrackBack()
	end

	return self:Stmt7()
end

function Parser:Stmt7()
	if self:AcceptRoamingToken("var") then
		local trace = self:GetTokenTrace()
		local var = self:GetTokenData()

		if self:AcceptRoamingToken("aadd") then
			return self:Instruction(trace, "ass", var, self:Instruction(trace, "add", self:Instruction(trace, "var", var), self:Expr1()))
		elseif self:AcceptRoamingToken("asub") then
			return self:Instruction(trace, "ass", var, self:Instruction(trace, "sub", self:Instruction(trace, "var", var), self:Expr1()))
		elseif self:AcceptRoamingToken("amul") then
			return self:Instruction(trace, "ass", var, self:Instruction(trace, "mul", self:Instruction(trace, "var", var), self:Expr1()))
		elseif self:AcceptRoamingToken("adiv") then
			return self:Instruction(trace, "ass", var, self:Instruction(trace, "div", self:Instruction(trace, "var", var), self:Expr1()))
		end

		self:TrackBack()
	end

	return self:Stmt8()
end

function Parser:Stmt8()
	if self:AcceptRoamingToken("var") then
		local tbpos = self.index
		local trace = self:GetTokenTrace()
		local var = self:GetTokenData()

		if self:AcceptTailingToken("lsb") then
			if self:AcceptRoamingToken("rsb") then
				self:Error("Indexing operator ([]) requires an index [X]")
			end

			local expr = self:Expr1()
			if self:AcceptRoamingToken("com") then
				if !self:AcceptRoamingToken("fun") then
					self:Error("Indexing operator ([]) requires a lower case type [X,t]")
				end

				local longtp = self:GetTokenData()

				if !self:AcceptRoamingToken("rsb") then
					self:Error("Right square bracket (]) missing, to close indexing operator [X,t]")
				end

				if self:AcceptRoamingToken("ass") then
					if longtp == "number" then longtp = "normal" end
					if wire_expression_types[string.upper(longtp)] == nil then
						self:Error("Indexing operator ([]) does not support the type [" .. longtp .. "]")
					end

					local tp = wire_expression_types[string.upper(longtp)][1]
					return self:Instruction(trace, "set", var, expr, self:Stmt7(), tp)
				end
			elseif self:AcceptRoamingToken("rsb") then
				if self:AcceptRoamingToken("ass") then
					return self:Instruction(trace, "set", var, expr, self:Stmt7())
				end
			else
				self:Error("Indexing operator ([]) needs to be closed with comma (,) or right square bracket (])")
			end
		elseif self:AcceptRoamingToken("lsb") then
			self:Error("Indexing operator ([]) must not be preceded by whitespace")
		elseif self:AcceptRoamingToken("ass") then
			return self:Instruction(trace, "ass", var, self:Stmt7())
		end

		self.index = tbpos - 2<commit_msg>[E2] Fixed being able to do X = Y++<commit_after>/******************************************************************************\
  Expression 2 Parser for Garry's Mod
  Andreas "Syranide" Svensson, <EMAIL>
\******************************************************************************/

AddCSLuaFile("parser.lua")

/*

Root
 1 : q1

seQuencing
 1 : ""
 2 : "s1 q1", "s1, q2"

Statement
 1 : if (e1) { q1 } i1
 2 : while (e1) { q1 }
 3 : for (var = e1, e1[, e1]) { q1 }
 4 : foreach(var, var:type = e1) { q1}
 5 : break, continue
 6 : var++, var--
 7 : var += e1, var -= e1, var *= e1, var /= e1
 8 : var = s8, var[e1,type] = s8
 9 : e1

If
 1 : elseif (e1) { q1 } i1
 2 : else { q1 }

Expression
 1 : var = e1, var += e1, var -= e1, var *= e1, var /= e1 [ERROR]
 2 : e3 ? e1 : e1, e3 ?: e1
 3 : e3 | e4
 4 : e4 & e5
 5 : e5 == e6, e5 != e6
 6 : e6 < e7, e6 > e7, e6 <= e7, e6 >= e7
 7 : e7 + e8, e7 - e8
 8 : e8 * e9, e8 / e9, e8 % e9
 9 : e9 ^ e10
10 : +e11, -e11, !e10
11 : e11:fun([e1, ...]), e11[var,type]
12 : (e1), fun([e1, ...])
13 : string, num, ~var, $var, ->var
14 : var++, var-- [ERROR]
15 : var

*/
/******************************************************************************/

Parser = {}
Parser.__index = Parser

function Parser.Execute(...)
	-- instantiate Parser
	local instance = setmetatable({}, Parser)

	-- and pcall the new instance's Process method.
	return pcall(Parser.Process, instance, ...)
end

function Parser:Error(message, token)
	if token then
		error(message .. " at line " .. token[4] .. ", char " .. token[5], 0)
	else
		error(message .. " at line " .. self.token[4] .. ", char " .. self.token[5], 0)
	end
end

function Parser:Process(tokens, params)
	self.tokens = tokens

	self.index = 0
	self.count = #tokens
	self.delta = {}

	self:NextToken()
	local tree = self:Root()

	return tree, self.delta
end

/******************************************************************************/

function Parser:GetToken()
	return self.token
end

function Parser:GetTokenData()
	return self.token[2]
end

function Parser:GetTokenTrace()
	return {self.token[4], self.token[5]}
end


function Parser:Instruction(trace, name, ...)
	return {name, trace, ...} //
end


function Parser:HasTokens()
	return self.readtoken != nil
end

function Parser:NextToken()
	if self.index <= self.count then
		if self.index > 0 then
			self.token = self.readtoken
		else
			self.token = {"", "", false, 1, 1}
		end

		self.index = self.index + 1
		self.readtoken = self.tokens[self.index]
	else
		self.readtoken = nil
	end
end

function Parser:TrackBack()
	self.index = self.index - 2
	self:NextToken()
end


function Parser:AcceptRoamingToken(name)
	local token = self.readtoken
	if !token or token[1] != name then return false end

	self:NextToken()
	return true
end

function Parser:AcceptTailingToken(name)
	local token = self.readtoken
	if !token or token[3] then return false end

	return self:AcceptRoamingToken(name)
end

function Parser:AcceptLeadingToken(name)
	local token = self.tokens[self.index + 1]
	if !token or token[3] then return false end

	return self:AcceptRoamingToken(name)
end


function Parser:RecurseLeft(func, tbl)
	local expr = func(self)
	local hit = true

	while hit do
		hit = false
		for i=1,#tbl do
			if self:AcceptRoamingToken(tbl[i]) then
				local trace = self:GetTokenTrace()

				hit = true
				expr = self:Instruction(trace, tbl[i], expr, func(self))
				break
			end
		end
	end

	return expr
end

/******************************************************************************/

local loopdepth

function Parser:Root()
	loopdepth = 0
	return self:Stmts()
end


function Parser:Stmts()
	local trace = self:GetTokenTrace()
	local stmts = self:Instruction(trace, "seq")

	if !self:HasTokens() then return stmts end

	while true do
		if self:AcceptRoamingToken("com") then
			self:Error("Statement separator (,) must not appear multiple times")
		end

		stmts[#stmts + 1] = self:Stmt1()

		if !self:HasTokens() then break end

		if !self:AcceptRoamingToken("com") then
			if self.readtoken[3] == false then
				self:Error("Statements must be separated by comma (,) or whitespace")
			end
		end
	end

	return stmts
end


function Parser:Stmt1()
	if self:AcceptRoamingToken("if") then
		local trace = self:GetTokenTrace()
		return self:Instruction(trace, "if", self:Cond(), self:Block("if condition"), self:IfElseIf())
	end

	return self:Stmt2()
end

function Parser:Stmt2()
	if self:AcceptRoamingToken("whl") then
		local trace = self:GetTokenTrace()
		loopdepth = loopdepth + 1
		local whl = self:Instruction(trace, "whl", self:Cond(), self:Block("while condition"))
		loopdepth = loopdepth - 1
		return whl
	end

	return self:Stmt3()
end

function Parser:Stmt3()
	if self:AcceptRoamingToken("for") then
		local trace = self:GetTokenTrace()
		loopdepth = loopdepth + 1

		if !self:AcceptRoamingToken("lpa") then
			self:Error("Left parenthesis (() must appear before condition")
		end

		if !self:AcceptRoamingToken("var") then
			self:Error("Variable expected for the numeric index")
		end

		local var = self:GetTokenData()

		if !self:AcceptRoamingToken("ass") then
			self:Error("Assignment operator (=) expected to preceed variable")
		end

		local estart = self:Expr1()

		if !self:AcceptRoamingToken("com") then
			self:Error("Comma (,) expected after start value")
		end

		local estop = self:Expr1()

		local estep
		if self:AcceptRoamingToken("com") then
			estep = self:Expr1()
		end

		if !self:AcceptRoamingToken("rpa") then
			self:Error("Right parenthesis ()) missing, to close condition")
		end

		local sfor = self:Instruction(trace, "for", var, estart, estop, estep, self:Block("for statement"))

		loopdepth = loopdepth - 1
		return sfor
	end

	return self:Stmt4()
end

function Parser:Stmt4()
	if self:AcceptRoamingToken("fea") then
		local trace = self:GetTokenTrace()
		loopdepth = loopdepth + 1

		if not self:AcceptRoamingToken("lpa") then
			self:Error("Left parenthesis missing (() after foreach statement")
		end

		if not self:AcceptRoamingToken("var") then
			self:Error("Variable expected to hold the key")
		end
		local keyvar = self:GetTokenData()

		if not self:AcceptRoamingToken("com") then
			self:Error("Comma (,) expected after key variable")
		end

		if not self:AcceptRoamingToken("var") then
			self:Error("Variable expected to hold the value")
		end
		local valvar = self:GetTokenData()

		if not self:AcceptRoamingToken("col") then
			self:Error("Colon (:) expected to separate type from variable")
		end

		if not self:AcceptRoamingToken("fun") and not self:AcceptRoamingToken("udf") then
			self:Error("Type expected after colon")
		end
		local valtype = self:GetTokenData()
		if valtype == "number" then valtype = "normal" end
		if wire_expression_types[string.upper(valtype)] == nil then
			self:Error("Unknown type: "..valtype)
		end
		valtype = wire_expression_types[string.upper(valtype)][1]

		if not self:AcceptRoamingToken("ass") then
			self:Error("Equals sign (=) expected after value type to specify table")
		end

		local tableexpr = self:Expr1()

		if not self:AcceptRoamingToken("rpa") then
			self:Error("Missing right parenthesis after foreach statement")
		end

		local sfea = self:Instruction(trace, "fea", keyvar, valvar, valtype, tableexpr, self:Block("foreach statement"))
		loopdepth = loopdepth -1
		return sfea
	end

	return self:Stmt5()
end

function Parser:Stmt5()
	if self:AcceptRoamingToken("brk") then
		if loopdepth > 0 then
			local trace = self:GetTokenTrace()
			return self:Instruction(trace, "brk")
		else
			self:Error("Break may not exist outside of a loop")
		end
	elseif self:AcceptRoamingToken("cnt") then
		if loopdepth > 0 then
			local trace = self:GetTokenTrace()
			return self:Instruction(trace, "cnt")
		else
			self:Error("Continue may not exist outside of a loop")
		end
	end

	return self:Stmt6()
end

function Parser:Stmt6()
	if self:AcceptRoamingToken("var") then
		local trace = self:GetTokenTrace()
		local var = self:GetTokenData()

		if self:AcceptTailingToken("inc") then
			return self:Instruction(trace, "inc", var)
		elseif self:AcceptRoamingToken("inc") then
			self:Error("Increment operator (++) must not be preceded by whitespace")
		end

		if self:AcceptTailingToken("dec") then
			return self:Instruction(trace, "dec", var)
		elseif self:AcceptRoamingToken("dec") then
			self:Error("Decrement operator (--) must not be preceded by whitespace")
		end

		self:TrackBack()
	end

	return self:Stmt7()
end

function Parser:Stmt7()
	if self:AcceptRoamingToken("var") then
		local trace = self:GetTokenTrace()
		local var = self:GetTokenData()

		if self:AcceptRoamingToken("aadd") then
			return self:Instruction(trace, "ass", var, self:Instruction(trace, "add", self:Instruction(trace, "var", var), self:Expr1()))
		elseif self:AcceptRoamingToken("asub") then
			return self:Instruction(trace, "ass", var, self:Instruction(trace, "sub", self:Instruction(trace, "var", var), self:Expr1()))
		elseif self:AcceptRoamingToken("amul") then
			return self:Instruction(trace, "ass", var, self:Instruction(trace, "mul", self:Instruction(trace, "var", var), self:Expr1()))
		elseif self:AcceptRoamingToken("adiv") then
			return self:Instruction(trace, "ass", var, self:Instruction(trace, "div", self:Instruction(trace, "var", var), self:Expr1()))
		end

		self:TrackBack()
	end

	return self:Stmt8()
end

function Parser:Stmt8()
	if self:AcceptRoamingToken("var") then
		local tbpos = self.index
		local trace = self:GetTokenTrace()
		local var = self:GetTokenData()

		if self:AcceptTailingToken("lsb") then
			if self:AcceptRoamingToken("rsb") then
				self:Error("Indexing operator ([]) requires an index [X]")
			end

			local expr = self:Expr1()
			if self:AcceptRoamingToken("com") then
				if !self:AcceptRoamingToken("fun") then
					self:Error("Indexing operator ([]) requires a lower case type [X,t]")
				end

				local longtp = self:GetTokenData()

				if !self:AcceptRoamingToken("rsb") then
					self:Error("Right square bracket (]) missing, to close indexing operator [X,t]")
				end

				if self:AcceptRoamingToken("ass") then
					if longtp == "number" then longtp = "normal" end
					if wire_expression_types[string.upper(longtp)] == nil then
						self:Error("Indexing operator ([]) does not support the type [" .. longtp .. "]")
					end

					local tp = wire_expression_types[string.upper(longtp)][1]
					return self:Instruction(trace, "set", var, expr, self:Stmt8(), tp)
				end
			elseif self:AcceptRoamingToken("rsb") then
				if self:AcceptRoamingToken("ass") then
					return self:Instruction(trace, "set", var, expr, self:Stmt8())
				end
			else
				self:Error("Indexing operator ([]) needs to be closed with comma (,) or right square bracket (])")
			end
		elseif self:AcceptRoamingToken("lsb") then
			self:Error("Indexing operator ([]) must not be preceded by whitespace")
		elseif self:AcceptRoamingToken("ass") then
			return self:Instruction(trace, "ass", var, self:Stmt8())
		end

		self.index = tbpos - 2
{'fraction_non_alphanumeric': 0.16322624743677375, 'fraction_numerical': 0.016267942583732056, 'mean_word_length': 4.397442203639941, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2645566', 'n_tokens_mistral': 8015, 'n_tokens_neox': 7246, 'n_words': 2097}