<commit_before>local re = require 're'

-- copied from https://en.wikipedia.org/wiki/Escape_sequences_in_C
local escaped_char_map = {
	a = 0x07,
	b = 0x08,
	f = 0x0C,
	n = 0x0A,
	r = 0x0D,
	t = 0x09,
	v = 0x0B,
	["'"] = 0x27,
	['"'] = 0x22
}

local reverse_escaped_char_map = {}
for key, value in pairs(escaped_char_map) do
	reverse_escaped_char_map[value] = key
end

local function readable_char(char, type)
	local str = string.char(char)

	if str == "'" and type == 'char' then
		return [[\']]
	elseif str == '"' and type == 'string' then
		return [[\"]]
	end

	if str:find("%g") then
		return ("%s"):format(str)
	else
		if reverse_escaped_char_map[char] then
			return ("\\%s"):format(reverse_escaped_char_map[char])
		end

		return ("\\x%02X"):format(char)
	end
end

local metatables = {
	integer = {
		__index = {
			tostring = function(self)
				return ("%d"):format(self.value)
			end,
			size = function() return 4 end
		}
	},
	float = {
		__index = {
			tostring = function(self)
				return ("%g"):format(self.value)
			end,
			size = function() return 4 end
		}
	},
	character = {
		__index = {
			tostring = function(self)
				return ("'%s'"):format(readable_char(self.value, 'char'))
			end,
			size = function() return 1 end
		}
	},
	string = {
		__index = {
			tostring = function(self)
				return ('"%s"'):format(self.value:gsub(".", function(char)
					if char == '"' then
						return '\\"'
					end

					if char:find "%g" then
						return char
					else
						local byte = string.byte(char)
						if reverse_escaped_char_map[byte] then
							return "\\" .. reverse_escaped_char_map[byte]
						else
							return ("\\x%02X"):format(byte)
						end
					end
				end))
			end,
			size = function() return #self.value + 1 end
		}
	}
}

local function create_value(type, value)
	return setmetatable({type = type, value = value}, metatables[type])
end

local grammar = re.compile([[
	definitions <- {| (S definition S)+ |}
	definition <- function_definition / global_variable_definition
	global_variable_definition <- {| {:definition: '' -> 'global' :} <vardef> |} 
	static_initializer <- '=' S {: literal_value :}

	-- literal value
	literal_value <- float / integer / character / string
	integer <- hexadecimal_integer / decimal_integer
	hexadecimal_integer <- ('0x' HEXCHAR+) -> hexadecimal_integer
	decimal_integer <- (%d+) -> decimal_integer
	float <- (%d+ '.' %d+) -> float
	character <- "'" single_character "'"
	single_character <- ('\' { [abfnrtv'] } ) -> escaped_char / [^'] -> normal_char 
	string <- '"' {| char_in_string* |} -> string '"'
	char_in_string <- ('\' {[abfnrtv"]} ) -> escaped_char_map / [^"] -> string_byte

	function_definition <- {| {:definition:'' -> 'function' :} <function_head> {:body:function_body:} |}
	function_head <- {:return_type:RETURN_TYPE:} S {:name:IDENTIFIER:} S '(' S {:parameters:parameters:} S ')'
	function_body <- (S compound_statement S) -> flat_compound

	expression <- p1_expression
	primary_expression <- 
		literal_value 
		/ variable 
		/ PAREN_L expression PAREN_R
	p1_expression <- 
		{| {:primary:primary_expression:} {:tails: {| p1_tail+ |} :}? |} -> p1_tree
	p1_tail <- 
		{| {:tail: '' -> 'subscript' :} BRACKET_L {:subscript:expression:} BRACKET_R |}
		/ {| {:tail: '' -> 'dot' :} S '.' S {:member:IDENTIFIER:} |}
		/ {| {:tail: '' -> 'arrow':} S '->' S {:member:IDENTIFIER:} |}
		/ {| {:tail: '' -> 'call':} PAREN_L {:arguments:arguments:} PAREN_R |} 
		/ {| {:tail: '++' -> 'increment' :} |}
		/ {| {:tail: '--' -> 'decrement' :} |}

	function_call <- {| {:function_name:IDENTIFIER:} S {:arguments: '(' S {: arguments :} S ')' :} |}
	variable <- {| {:name: IDENTIFIER :} |} -> variable

	arguments <- {| argument (S ',' S argument)* |} / S
	argument <- expression

	-- statements
	statement <- compound_statement / jump_statement / expression_statement / vardef_statement / iteration_statement
	compound_statement <- {| {:statement: '' -> 'compound' :} BRACE_L {:statements:statements:} BRACE_R |}
	expression_statement <- {| {:statement: '' -> 'expression' :} {:expression: expression :} ENDING_SEMICOLON |}
	iteration_statement <- {| {:statement: '' -> 'iteration' :} <iteration_while> |}
	statements <- {| statement* |}

	iteration_while <- {:iteration: 'while' :} PAREN_L {:condition:expression:} PAREN_R {:body:statement:}
	<commit_msg>arguments is optional for function call AST<commit_after>local re = require 're'

-- copied from https://en.wikipedia.org/wiki/Escape_sequences_in_C
local escaped_char_map = {
	a = 0x07,
	b = 0x08,
	f = 0x0C,
	n = 0x0A,
	r = 0x0D,
	t = 0x09,
	v = 0x0B,
	["'"] = 0x27,
	['"'] = 0x22
}

local reverse_escaped_char_map = {}
for key, value in pairs(escaped_char_map) do
	reverse_escaped_char_map[value] = key
end

local function readable_char(char, type)
	local str = string.char(char)

	if str == "'" and type == 'char' then
		return [[\']]
	elseif str == '"' and type == 'string' then
		return [[\"]]
	end

	if str:find("%g") then
		return ("%s"):format(str)
	else
		if reverse_escaped_char_map[char] then
			return ("\\%s"):format(reverse_escaped_char_map[char])
		end

		return ("\\x%02X"):format(char)
	end
end

local metatables = {
	integer = {
		__index = {
			tostring = function(self)
				return ("%d"):format(self.value)
			end,
			size = function() return 4 end
		}
	},
	float = {
		__index = {
			tostring = function(self)
				return ("%g"):format(self.value)
			end,
			size = function() return 4 end
		}
	},
	character = {
		__index = {
			tostring = function(self)
				return ("'%s'"):format(readable_char(self.value, 'char'))
			end,
			size = function() return 1 end
		}
	},
	string = {
		__index = {
			tostring = function(self)
				return ('"%s"'):format(self.value:gsub(".", function(char)
					if char == '"' then
						return '\\"'
					end

					if char:find "%g" then
						return char
					else
						local byte = string.byte(char)
						if reverse_escaped_char_map[byte] then
							return "\\" .. reverse_escaped_char_map[byte]
						else
							return ("\\x%02X"):format(byte)
						end
					end
				end))
			end,
			size = function() return #self.value + 1 end
		}
	}
}

local function create_value(type, value)
	return setmetatable({type = type, value = value}, metatables[type])
end

local grammar = re.compile([[
	definitions <- {| (S definition S)+ |}
	definition <- function_definition / global_variable_definition
	global_variable_definition <- {| {:definition: '' -> 'global' :} <vardef> |} 
	static_initializer <- '=' S {: literal_value :}

	-- literal value
	literal_value <- float / integer / character / string
	integer <- hexadecimal_integer / decimal_integer
	hexadecimal_integer <- ('0x' HEXCHAR+) -> hexadecimal_integer
	decimal_integer <- (%d+) -> decimal_integer
	float <- (%d+ '.' %d+) -> float
	character <- "'" single_character "'"
	single_character <- ('\' { [abfnrtv'] } ) -> escaped_char / [^'] -> normal_char 
	string <- '"' {| char_in_string* |} -> string '"'
	char_in_string <- ('\' {[abfnrtv"]} ) -> escaped_char_map / [^"] -> string_byte

	function_definition <- {| {:definition:'' -> 'function' :} <function_head> {:body:function_body:} |}
	function_head <- {:return_type:RETURN_TYPE:} S {:name:IDENTIFIER:} S '(' S {:parameters:parameters:} S ')'
	function_body <- (S compound_statement S) -> flat_compound

	expression <- p1_expression
	primary_expression <- 
		literal_value 
		/ variable 
		/ PAREN_L expression PAREN_R
	p1_expression <- 
		{| {:primary:primary_expression:} {:tails: {| p1_tail+ |} :}? |} -> p1_tree
	p1_tail <- 
		{| {:tail: '' -> 'subscript' :} BRACKET_L {:subscript:expression:} BRACKET_R |}
		/ {| {:tail: '' -> 'dot' :} S '.' S {:member:IDENTIFIER:} |}
		/ {| {:tail: '' -> 'arrow':} S '->' S {:member:IDENTIFIER:} |}
		/ {| {:tail: '' -> 'call':} PAREN_L {:arguments:arguments:}? PAREN_R |} 
		/ {| {:tail: '++' -> 'increment' :} |}
		/ {| {:tail: '--' -> 'decrement' :} |}

	function_call <- {| {:function_name:IDENTIFIER:} S {:arguments: '(' S {: arguments :} S ')' :} |}
	variable <- {| {:name: IDENTIFIER :} |} -> variable

	arguments <- {| argument (S ',' S argument)* |}
	argument <- expression

	-- statements
	statement <- compound_statement / jump_statement / expression_statement / vardef_statement / iteration_statement
	compound_statement <- {| {:statement: '' -> 'compound' :} BRACE_L {:statements:statements:} BRACE_R |}
	expression_statement <- {| {:statement: '' -> 'expression' :} {:expression: expression :} ENDING_SEMICOLON |}
	iteration_statement <- {| {:statement: '' -> 'iteration' :} <iteration_while> |}
	statements <- {| statement* |}

	iteration_while <- {:iteration: 'while' :} PAREN_L {:condition:expression:} PAREN_R {:body:statement:}
	
{'fraction_non_alphanumeric': 0.18622360678435446, 'fraction_numerical': 0.00853813314872505, 'mean_word_length': 3.6303418803418803, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 55, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '184875', 'n_tokens_mistral': 3587, 'n_tokens_neox': 3107, 'n_words': 649}