<commit_before>--[[--------------------------------------------------------------------
    Copyright (C) 2014 <NAME>.
    See the file LICENSE.txt for copying permission.
--]]--------------------------------------------------------------------

--[[----------------------------------------------------------------------------
	This module implements a parser that generates an abstract syntax tree (AST)
	from an Ovale script.

	An AST data structure is a table with the following public properties:

		ast.annotation
		ast.annotation.customFunction
		ast.annotation.definition
		ast.annotation.functionCall
		ast.child
--]]----------------------------------------------------------------------------

local OVALE, Ovale = ...
local OvaleAST = Ovale:NewModule("OvaleAST")
Ovale.OvaleAST = OvaleAST

--<private-static-properties>
local L = Ovale.L
local OvalePool = Ovale.OvalePool
local OvaleProfiler = Ovale.OvaleProfiler

-- Forward declarations for module dependencies.
local OvaleCondition = nil
local OvaleLexer = nil
local OvaleScripts = nil
local OvaleSpellBook = nil

local format = string.format
local gsub = string.gsub
local ipairs = ipairs
local next = next
local pairs = pairs
local rawset = rawset
local setmetatable = setmetatable
local strlower = string.lower
local strmatch = string.match
local strsub = string.sub
local tconcat = table.concat
local tinsert = table.insert
local tonumber = tonumber
local tostring = tostring
local type = type
local wipe = table.wipe
local yield = coroutine.yield
local API_GetItemInfo = GetItemInfo

-- Register for profiling.
OvaleProfiler:RegisterProfiling(OvaleAST)

-- Keywords for the Ovale script language.
local KEYWORD = {
	["and"] = true,
	["if"] = true,
	["not"] = true,
	["or"] = true,
	["unless"] = true,
	["wait"] = true,
}

local DECLARATION_KEYWORD = {
	["AddActionIcon"] = true,
	["AddCheckBox"] = true,
	["AddFunction"] = true,
	["AddIcon"] = true,
	["AddListItem"] = true,
	["Define"] = true,
	["Include"] = true,
	["ItemInfo"] = true,
	["ItemList"] = true,
	["ScoreSpells"] = true,
	["SpellInfo"] = true,
	["SpellList"] = true,
	["SpellRequire"] = true,
}

local PARAMETER_KEYWORD = {
	["checkbox"] = true,
	["checkboxoff"] = true,		-- deprecated
	["checkboxon"] = true,		-- deprecated
	["glyph"] = true,
	["help"] = true,
	["if_spell"] = true,
	["if_stance"] = true,
	["item"] = true,			-- deprecated
	["itemcount"] = true,
	["itemset"] = true,
	["level"] = true,
	["list"] = true,			-- deprecated
	["listitem"] = true,
	["specialization"] = true,
	["talent"] = true,
	["text"] = true,
	["wait"] = true,
}

local SPELL_AURA_KEYWORD = {
	["SpellAddBuff"] = true,
	["SpellAddDebuff"] = true,
	["SpellAddTargetBuff"] = true,
	["SpellAddTargetDebuff"] = true,
	["SpellDamageBuff"] = true,
	["SpellDamageDebuff"] = true,
}

do
	-- SpellAuraList keywords are declaration keywords.
	for keyword, value in pairs(SPELL_AURA_KEYWORD) do
		DECLARATION_KEYWORD[keyword] = value
	end
	-- All keywords are Ovale script keywords.
	for keyword, value in pairs(DECLARATION_KEYWORD) do
		KEYWORD[keyword] = value
	end
	for keyword, value in pairs(PARAMETER_KEYWORD) do
		KEYWORD[keyword] = value
	end
end

-- Table of pattern/tokenizer pairs for the Ovale script language.
local MATCHES = nil

-- Functions that are actions; ACTION_PARAMETER_COUNT[action] = number of required parameters
local ACTION_PARAMETER_COUNT = {
	["item"] = 1,
	["macro"] = 1,
	["spell"] = 1,
	["texture"] = 1,
	["setstate"] = 2,
}

-- Actions that are special "state" actions and return no other relevant action information.
local STATE_ACTION = {
	["setstate"] = true,
}
-- Functions for accessing string databases.
local STRING_LOOKUP_FUNCTION = {
	["ItemName"] = true,
	["L"] = true,
	["SpellName"] = true,
}

-- Unary and binary operators with precedence.
local UNARY_OPERATOR = {
	["not"] = { "logical", 15 },
	["-"]   = { "arithmetic", 50 },
}
local BINARY_OPERATOR = {
	-- logical
	["or"]  = { "logical", 5, "associative" },
	["xor"]	= { "logical", 8, "associative" },
	["and"] = { "logical", 10, "associative" },
	-- comparison
	["!="]  = { "compare", 20 },
	["<"]   = { "compare", 20 },
	["<="]  = { "compare", 20 },
	["=="]  = { "compare", 20 },
	[">"]   = { "compare", 20 },
	[">="]  = { "compare", 20 },
	-- addition, subtraction
	["+"]   = { "arithmetic", 30, "associative" },
	["-"]   = { "arithmetic", 30 },
	-- multiplication, division, modulus
	["%"]   = { "arithmetic", 40 },
	["*"]   = { "arithmetic", 40, "associative" },
	["/"]   = { "arithmetic", 40 },
	-- exponentiation
	["^"]   = { "arithmetic", 100 },
}

-- INDENT[k] is a string of k concatenated tabs.
local INDENT = {}
do
	INDENT[0] = ""
	local metatable = {
		__index = function(tbl, key)
			key = tonumber(key)
			if key > 0 then
				local s = tbl[key - 1] .. "\t"
				rawset(tbl, key, s)
				return s
			end
			return INDENT[0]
		end,
	}
	setmetatable(INDENT, metatable)
end

local self_indent = 0
local self_outputPool = OvalePool("OvaleAST_outputPool")

local self_controlPool = OvalePool("OvaleAST_controlPool")
local self_parametersPool = OvalePool("OvaleAST_parametersPool")
local self_childrenPool = OvalePool("OvaleAST_childrenPool")
local self_pool = OvalePool("OvaleAST_pool")
do
	self_pool.Clean = function(self, node)
		if node.child then
			self_childrenPool:Release(node.child)
			node.child = nil
		end
	end
end
--</private-static-properties>

--<public-static-properties>
-- Export list of parameters keywords.
OvaleAST.PARAMETER_KEYWORD = PARAMETER_KEYWORD
--</public-static-properties>

--<private-static-methods>
-- Implementation of PHP-like print_r() taken from http://lua-users.org/wiki/TableSerialization.
-- This is used to print out a table, but has been modified to print out an AST.
local function print_r(node, indent, done, output)
	done = done or {}
	output = output or {}
	indent = indent or ''
	for key, value in pairs(node) do
		if type(value) == "table" then
			if done[value] then
				tinsert(output, indent .. "[" .. tostring(key) .. "] => (self_reference)")
			else
				-- Shortcut conditional allocation
				done[value] = true
				if value.type then
					tinsert(output, indent .. "[" .. tostring(key) .. "] =>")
				else
					tinsert(output, indent .. "[" .. tostring(key) .. "] => {")
				end
				print_r(value, indent .. "    ", done, output)
				if not value.type then
					tinsert(output, indent .. "}")
				end
			end
		else
			tinsert(output, indent .. "[" .. tostring(key) .. "] => " .. tostring(value))
		end
	end
	return output
end

-- Follow the flyweight pattern for number nodes.
local function GetNumberNode(value, nodeList, annotation)
	-- Check for a flyweight node with this exact numerical value.
	annotation.numberFlyweight = annotation.numberFlyweight or {}
	local node = annotation.numberFlyweight[value]
	if not node then
		node = OvaleAST:NewNode(nodeList)
		node.type = "value"
		node.value = value
		node.origin = 0
		node.rate = 0
		-- Store the first node with this exact numerical value in numberFlyweight.
		annotation.numberFlyweight[value] = node
	end
	return node
end

--[[---------------------------------------------
	Lexer functions (for use with OvaleLexer)
--]]---------------------------------------------
local function TokenizeComment(token)
	return yield("comment", token)
end

local function TokenizeLua(token, options)
	-- Strip off leading [[ and trailing ]].
	token = strsub(token, 3, -3)
	return yield("lua", token)
end

local function TokenizeName(token)
	if KEYWORD[token] then
		return yield("keyword", token)
	else
		return yield("name", token)
	end
end

local function TokenizeNumber(token, options)
	if options and options.number then
		token = tonumber(token)
	end
	return yield("number", token)
end

local function TokenizeString(token, options)
	-- Strip leading and trailing quote characters.
	if options and options.string then
		token = strsub(token, 2, -2)
	end
	return yield("string", token)
end

local function TokenizeWhitespace(token)
	return yield("space", token)
end

local function Tokenize(token)
	return yield(token, token)
end

local function NoToken()
	return yield(nil)
end

do
	MATCHES = {
		{ "^%s+", TokenizeWhitespace },
		{ "^%d+%.?%d*", TokenizeNumber },
		{ "^[%a_][%w_]*", TokenizeName },
		{ "^((['\"])%2)", TokenizeString },	-- empty string
		{ [[^(['\"]).-\\%1]], TokenizeString },
		{ [[^(['\"]).-[^\]%1]], TokenizeString },
		{ "^#.-\n", TokenizeComment },
		{ "^!=", Tokenize },
		{ "^==", Tokenize },
		{ "^<=", Tokenize },
		{ "^>=", Tokenize },
		{ "^.", Tokenize },
		{ "^$", NoToken },
	}
end

local function GetTokenIterator(s)
	local exclude = { space = true, comments = true }
	do
		-- Fix some API brokenness in the Penlight lexer.
		if exclude.space then
			exclude[TokenizeWhitespace] = true
		end
		if exclude.comments then
			exclude[TokenizeComment] = true
		end
	end
	return OvaleLexer.scan(s, MATCHES, exclude)
end

-- "Flatten" a parameter value node into a string.
local function FlattenParameterValue(parameterValue)
	local value = parameterValue
	if type(parameterValue) == "table" then
		local node = parameterValue
		local isBang = false
		if node.type == "bang_value" then
			isBang = true
			node = node.child[1]
		end
		if node.type == "value" then
			value = node.value
		elseif node.type == "variable" then
			value = node.name
		elseif node.type == "string" then
			value = node.value
		elseif node.type == "comma_separated_values" then
			local output = self_outputPool:Get()
			for k, v in ipairs(node.csv) do
				output[k] = FlattenParameterValue(v)
			end
			value = tconcat(output, ",")
			self_outputPool:Release(output)
		end
		if isBang then
			value = "!" .. tostring(value)
		end
	end<commit_msg>Properly flatten parameters into strings when "!" is used.<commit_after>--[[--------------------------------------------------------------------
    Copyright (C) 2014 <NAME>.
    See the file LICENSE.txt for copying permission.
--]]--------------------------------------------------------------------

--[[----------------------------------------------------------------------------
	This module implements a parser that generates an abstract syntax tree (AST)
	from an Ovale script.

	An AST data structure is a table with the following public properties:

		ast.annotation
		ast.annotation.customFunction
		ast.annotation.definition
		ast.annotation.functionCall
		ast.child
--]]----------------------------------------------------------------------------

local OVALE, Ovale = ...
local OvaleAST = Ovale:NewModule("OvaleAST")
Ovale.OvaleAST = OvaleAST

--<private-static-properties>
local L = Ovale.L
local OvalePool = Ovale.OvalePool
local OvaleProfiler = Ovale.OvaleProfiler

-- Forward declarations for module dependencies.
local OvaleCondition = nil
local OvaleLexer = nil
local OvaleScripts = nil
local OvaleSpellBook = nil

local format = string.format
local gsub = string.gsub
local ipairs = ipairs
local next = next
local pairs = pairs
local rawset = rawset
local setmetatable = setmetatable
local strlower = string.lower
local strmatch = string.match
local strsub = string.sub
local tconcat = table.concat
local tinsert = table.insert
local tonumber = tonumber
local tostring = tostring
local type = type
local wipe = table.wipe
local yield = coroutine.yield
local API_GetItemInfo = GetItemInfo

-- Register for profiling.
OvaleProfiler:RegisterProfiling(OvaleAST)

-- Keywords for the Ovale script language.
local KEYWORD = {
	["and"] = true,
	["if"] = true,
	["not"] = true,
	["or"] = true,
	["unless"] = true,
	["wait"] = true,
}

local DECLARATION_KEYWORD = {
	["AddActionIcon"] = true,
	["AddCheckBox"] = true,
	["AddFunction"] = true,
	["AddIcon"] = true,
	["AddListItem"] = true,
	["Define"] = true,
	["Include"] = true,
	["ItemInfo"] = true,
	["ItemList"] = true,
	["ScoreSpells"] = true,
	["SpellInfo"] = true,
	["SpellList"] = true,
	["SpellRequire"] = true,
}

local PARAMETER_KEYWORD = {
	["checkbox"] = true,
	["checkboxoff"] = true,		-- deprecated
	["checkboxon"] = true,		-- deprecated
	["glyph"] = true,
	["help"] = true,
	["if_spell"] = true,
	["if_stance"] = true,
	["item"] = true,			-- deprecated
	["itemcount"] = true,
	["itemset"] = true,
	["level"] = true,
	["list"] = true,			-- deprecated
	["listitem"] = true,
	["specialization"] = true,
	["talent"] = true,
	["text"] = true,
	["wait"] = true,
}

local SPELL_AURA_KEYWORD = {
	["SpellAddBuff"] = true,
	["SpellAddDebuff"] = true,
	["SpellAddTargetBuff"] = true,
	["SpellAddTargetDebuff"] = true,
	["SpellDamageBuff"] = true,
	["SpellDamageDebuff"] = true,
}

do
	-- SpellAuraList keywords are declaration keywords.
	for keyword, value in pairs(SPELL_AURA_KEYWORD) do
		DECLARATION_KEYWORD[keyword] = value
	end
	-- All keywords are Ovale script keywords.
	for keyword, value in pairs(DECLARATION_KEYWORD) do
		KEYWORD[keyword] = value
	end
	for keyword, value in pairs(PARAMETER_KEYWORD) do
		KEYWORD[keyword] = value
	end
end

-- Table of pattern/tokenizer pairs for the Ovale script language.
local MATCHES = nil

-- Functions that are actions; ACTION_PARAMETER_COUNT[action] = number of required parameters
local ACTION_PARAMETER_COUNT = {
	["item"] = 1,
	["macro"] = 1,
	["spell"] = 1,
	["texture"] = 1,
	["setstate"] = 2,
}

-- Actions that are special "state" actions and return no other relevant action information.
local STATE_ACTION = {
	["setstate"] = true,
}
-- Functions for accessing string databases.
local STRING_LOOKUP_FUNCTION = {
	["ItemName"] = true,
	["L"] = true,
	["SpellName"] = true,
}

-- Unary and binary operators with precedence.
local UNARY_OPERATOR = {
	["not"] = { "logical", 15 },
	["-"]   = { "arithmetic", 50 },
}
local BINARY_OPERATOR = {
	-- logical
	["or"]  = { "logical", 5, "associative" },
	["xor"]	= { "logical", 8, "associative" },
	["and"] = { "logical", 10, "associative" },
	-- comparison
	["!="]  = { "compare", 20 },
	["<"]   = { "compare", 20 },
	["<="]  = { "compare", 20 },
	["=="]  = { "compare", 20 },
	[">"]   = { "compare", 20 },
	[">="]  = { "compare", 20 },
	-- addition, subtraction
	["+"]   = { "arithmetic", 30, "associative" },
	["-"]   = { "arithmetic", 30 },
	-- multiplication, division, modulus
	["%"]   = { "arithmetic", 40 },
	["*"]   = { "arithmetic", 40, "associative" },
	["/"]   = { "arithmetic", 40 },
	-- exponentiation
	["^"]   = { "arithmetic", 100 },
}

-- INDENT[k] is a string of k concatenated tabs.
local INDENT = {}
do
	INDENT[0] = ""
	local metatable = {
		__index = function(tbl, key)
			key = tonumber(key)
			if key > 0 then
				local s = tbl[key - 1] .. "\t"
				rawset(tbl, key, s)
				return s
			end
			return INDENT[0]
		end,
	}
	setmetatable(INDENT, metatable)
end

local self_indent = 0
local self_outputPool = OvalePool("OvaleAST_outputPool")

local self_controlPool = OvalePool("OvaleAST_controlPool")
local self_parametersPool = OvalePool("OvaleAST_parametersPool")
local self_childrenPool = OvalePool("OvaleAST_childrenPool")
local self_pool = OvalePool("OvaleAST_pool")
do
	self_pool.Clean = function(self, node)
		if node.child then
			self_childrenPool:Release(node.child)
			node.child = nil
		end
	end
end
--</private-static-properties>

--<public-static-properties>
-- Export list of parameters keywords.
OvaleAST.PARAMETER_KEYWORD = PARAMETER_KEYWORD
--</public-static-properties>

--<private-static-methods>
-- Implementation of PHP-like print_r() taken from http://lua-users.org/wiki/TableSerialization.
-- This is used to print out a table, but has been modified to print out an AST.
local function print_r(node, indent, done, output)
	done = done or {}
	output = output or {}
	indent = indent or ''
	for key, value in pairs(node) do
		if type(value) == "table" then
			if done[value] then
				tinsert(output, indent .. "[" .. tostring(key) .. "] => (self_reference)")
			else
				-- Shortcut conditional allocation
				done[value] = true
				if value.type then
					tinsert(output, indent .. "[" .. tostring(key) .. "] =>")
				else
					tinsert(output, indent .. "[" .. tostring(key) .. "] => {")
				end
				print_r(value, indent .. "    ", done, output)
				if not value.type then
					tinsert(output, indent .. "}")
				end
			end
		else
			tinsert(output, indent .. "[" .. tostring(key) .. "] => " .. tostring(value))
		end
	end
	return output
end

-- Follow the flyweight pattern for number nodes.
local function GetNumberNode(value, nodeList, annotation)
	-- Check for a flyweight node with this exact numerical value.
	annotation.numberFlyweight = annotation.numberFlyweight or {}
	local node = annotation.numberFlyweight[value]
	if not node then
		node = OvaleAST:NewNode(nodeList)
		node.type = "value"
		node.value = value
		node.origin = 0
		node.rate = 0
		-- Store the first node with this exact numerical value in numberFlyweight.
		annotation.numberFlyweight[value] = node
	end
	return node
end

--[[---------------------------------------------
	Lexer functions (for use with OvaleLexer)
--]]---------------------------------------------
local function TokenizeComment(token)
	return yield("comment", token)
end

local function TokenizeLua(token, options)
	-- Strip off leading [[ and trailing ]].
	token = strsub(token, 3, -3)
	return yield("lua", token)
end

local function TokenizeName(token)
	if KEYWORD[token] then
		return yield("keyword", token)
	else
		return yield("name", token)
	end
end

local function TokenizeNumber(token, options)
	if options and options.number then
		token = tonumber(token)
	end
	return yield("number", token)
end

local function TokenizeString(token, options)
	-- Strip leading and trailing quote characters.
	if options and options.string then
		token = strsub(token, 2, -2)
	end
	return yield("string", token)
end

local function TokenizeWhitespace(token)
	return yield("space", token)
end

local function Tokenize(token)
	return yield(token, token)
end

local function NoToken()
	return yield(nil)
end

do
	MATCHES = {
		{ "^%s+", TokenizeWhitespace },
		{ "^%d+%.?%d*", TokenizeNumber },
		{ "^[%a_][%w_]*", TokenizeName },
		{ "^((['\"])%2)", TokenizeString },	-- empty string
		{ [[^(['\"]).-\\%1]], TokenizeString },
		{ [[^(['\"]).-[^\]%1]], TokenizeString },
		{ "^#.-\n", TokenizeComment },
		{ "^!=", Tokenize },
		{ "^==", Tokenize },
		{ "^<=", Tokenize },
		{ "^>=", Tokenize },
		{ "^.", Tokenize },
		{ "^$", NoToken },
	}
end

local function GetTokenIterator(s)
	local exclude = { space = true, comments = true }
	do
		-- Fix some API brokenness in the Penlight lexer.
		if exclude.space then
			exclude[TokenizeWhitespace] = true
		end
		if exclude.comments then
			exclude[TokenizeComment] = true
		end
	end
	return OvaleLexer.scan(s, MATCHES, exclude)
end

-- "Flatten" a parameter value node into a string.
local function FlattenParameterValue(parameterValue)
	local value = parameterValue
	if type(parameterValue) == "table" then
		local node = parameterValue
		if node.type == "comma_separated_values" then
			local output = self_outputPool:Get()
			for k, v in ipairs(node.csv) do
				output[k] = FlattenParameterValue(v)
			end
			value = tconcat(output, ",")
			self_outputPool:Release(output)
		else
			local isBang = false
			if node.type == "bang_value" then
				isBang = true
				node = node.child[1]
			end
			if node.type == "value" then
				value = node.value
			elseif node.type == "variable" then
				value = node.name
			elseif node.type == "string" then
				value = node.value
			end
			if isBang then
				value = "!" .. tostring(value)
			end
		end
	end
{'fraction_non_alphanumeric': 0.1831486730224169, 'fraction_numerical': 0.005874774542643649, 'mean_word_length': 3.186839266450917, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3148011', 'n_tokens_mistral': 6930, 'n_tokens_neox': 6412, 'n_words': 1776}