<commit_before>local tinsert = table.insert

local lexer = require "luacheck.lexer"

local function new_state(src)
   return {lexer = lexer.new_state(src)}
end

local function skip_token(state)
   repeat
      state.token, state.token_value, state.line, state.column, state.offset = lexer.next_token(state.lexer)
   until state.token ~= "TK_COMMENT"
end

local function location(state)
   return {
      line = state.line,
      column = state.column,
      offset = state.offset
   }
end

local function init_ast_node(node, location, tag)
   node.line = location.line
   node.column = location.column
   node.offset = location.offset
   node.tag = tag
   return node
end

local function new_ast_node(state, tag)
   return init_ast_node({}, state, tag)
end

local function check_token(state, token)
   if state.token ~= token then
      error({})
   end
end

local function check_and_skip_token(state, token)
   check_token(state, token)
   skip_token(state)
end

local function test_and_skip_token(state, token)
   if state.token == token then
      skip_token(state)
      return true
   end
end

local function check_name(state)
   check_token(state, "TK_NAME")
   return state.token_value
end

local parse_block, parse_expression

local function parse_expression_list(state)
   local list = {}
   local is_inside_parentheses

   repeat
      list[#list+1], is_inside_parentheses = parse_expression(state)
   until not test_and_skip_token(state, ",")

   -- Parentheses around the last expression may be significant.
   if is_inside_parentheses then
      local last = list[#list]
      local tag = last.tag
      if tag == "Call" or tag == "Invoke" or tag == "Dots" then
         list[#list] = init_ast_node({last}, last, "Paren")
      end
   end

   return list
end

local function parse_var(state)
   local ast_node = new_ast_node(state, "Id")
   ast_node[1] = check_name(state)
   skip_token(state)  -- Skip name.
   return ast_node
end

local function parse_name(state)
   local ast_node = new_ast_node(state, "String")
   ast_node[1] = check_name(state)
   skip_token(state)  -- Skip name.
   return ast_node
end

local function parse_number(state)
   local ast_node = new_ast_node(state, "Number")
   ast_node[1] = state.token_value
   skip_token(state)  -- Skip number.
   return ast_node
end

local function parse_string(state)
   local ast_node = new_ast_node(state, "String")
   ast_node[1] = state.token_value
   skip_token(state)  -- Skip string.
   return ast_node
end

local function parse_nil(state)
   local ast_node = new_ast_node(state, "Nil")
   skip_token(state)  -- Skip "nil".
   return ast_node
end

local function parse_true(state)
   local ast_node = new_ast_node(state, "True")
   skip_token(state)  -- Skip "true".
   return ast_node
end

local function parse_false(state)
   local ast_node = new_ast_node(state, "False")
   skip_token(state)  -- Skip "false".
   return ast_node
end

local function parse_dots(state)
   local ast_node = new_ast_node(state, "Dots")
   skip_token(state)  -- Skip "...".
   return ast_node
end

local function parse_table(state)
   local ast_node = new_ast_node(state, "Table")
   skip_token(state)  -- Skip "{"

   repeat
      if state.token == "}" then
         break
      else
         local lhs, rhs
         local item_location = location(state)

         if state.token == "TK_NAME" then
            local name = state.token_value
            skip_token(state)  -- Skip name.

            if test_and_skip_token(state, "=") then
               -- `name` = `expr`.
               lhs = init_ast_node({name}, item_location, "String")
               rhs = parse_expression(state)
            else
               -- `name` is beginning of an expression in array part.
               -- Backtrack lexer to before name.
               state.lexer.line = item_location.line
               state.lexer.line_offset = item_location.offset-item_location.column+1
               state.lexer.offset = item_location.offset
               skip_token(state)  -- Load name again.
               rhs = parse_expression(state)
            end
         elseif test_and_skip_token(state, "[") then
            -- [ `expr` ] = `expr`.
            lhs = parse_expression(state)
            check_and_skip_token(state, "]")
            check_and_skip_token(state, "=")
            rhs = parse_expression(state)
         else
            -- Expression in array part.
            rhs = parse_expression(state)
         end

         if lhs then
            -- Pair.
            ast_node[#ast_node+1] = init_ast_node({lhs, rhs}, item_location, "Pair")
         else
            -- Array part item.
            ast_node[#ast_node+1] = rhs
         end
      end
   until not (test_and_skip_token(state, ",") or test_and_skip_token(state, ";"))

   check_and_skip_token(state, "}")
   return ast_node
end

-- Parses argument list and the statements.
local function parse_function(state, location)
   check_and_skip_token(state, "(")
   local args = {}

   if state.token ~= ")" then  -- Are there arguments?
      repeat
         if state.token == "TK_NAME" then
            args[#args+1] = parse_var(state)
         elseif state.token == "TK_DOTS" then
            args[#args+1] = parse_dots(state)
            break
         else
            error({})
         end
      until not test_and_skip_token(state, ",")
   end

   check_and_skip_token(state, ")")
   local body = parse_block(state)
   check_and_skip_token(state, "TK_END")
   return init_ast_node({args, body}, location, "Function")
end
<commit_msg>Do not ignore significant parens in table constructors<commit_after>local tinsert = table.insert

local lexer = require "luacheck.lexer"

local function new_state(src)
   return {lexer = lexer.new_state(src)}
end

local function skip_token(state)
   repeat
      state.token, state.token_value, state.line, state.column, state.offset = lexer.next_token(state.lexer)
   until state.token ~= "TK_COMMENT"
end

local function location(state)
   return {
      line = state.line,
      column = state.column,
      offset = state.offset
   }
end

local function init_ast_node(node, location, tag)
   node.line = location.line
   node.column = location.column
   node.offset = location.offset
   node.tag = tag
   return node
end

local function new_ast_node(state, tag)
   return init_ast_node({}, state, tag)
end

local function check_token(state, token)
   if state.token ~= token then
      error({})
   end
end

local function check_and_skip_token(state, token)
   check_token(state, token)
   skip_token(state)
end

local function test_and_skip_token(state, token)
   if state.token == token then
      skip_token(state)
      return true
   end
end

local function check_name(state)
   check_token(state, "TK_NAME")
   return state.token_value
end

-- If needed, wraps last expression in expressions in "Paren" node.
local function opt_add_parens(expressions, is_inside_parentheses)
   if is_inside_parentheses then
      local last = expressions[#expressions]

      if last and last.tag == "Call" or last.tag == "Invoke" or last.tag == "Dots" then
         expressions[#expressions] = init_ast_node({last}, last, "Paren")
      end
   end
end

local parse_block, parse_expression

local function parse_expression_list(state)
   local list = {}
   local is_inside_parentheses = false

   repeat
      list[#list+1], is_inside_parentheses = parse_expression(state)
   until not test_and_skip_token(state, ",")

   opt_add_parens(list, is_inside_parentheses)
   return list
end

local function parse_var(state)
   local ast_node = new_ast_node(state, "Id")
   ast_node[1] = check_name(state)
   skip_token(state)  -- Skip name.
   return ast_node
end

local function parse_name(state)
   local ast_node = new_ast_node(state, "String")
   ast_node[1] = check_name(state)
   skip_token(state)  -- Skip name.
   return ast_node
end

local function parse_number(state)
   local ast_node = new_ast_node(state, "Number")
   ast_node[1] = state.token_value
   skip_token(state)  -- Skip number.
   return ast_node
end

local function parse_string(state)
   local ast_node = new_ast_node(state, "String")
   ast_node[1] = state.token_value
   skip_token(state)  -- Skip string.
   return ast_node
end

local function parse_nil(state)
   local ast_node = new_ast_node(state, "Nil")
   skip_token(state)  -- Skip "nil".
   return ast_node
end

local function parse_true(state)
   local ast_node = new_ast_node(state, "True")
   skip_token(state)  -- Skip "true".
   return ast_node
end

local function parse_false(state)
   local ast_node = new_ast_node(state, "False")
   skip_token(state)  -- Skip "false".
   return ast_node
end

local function parse_dots(state)
   local ast_node = new_ast_node(state, "Dots")
   skip_token(state)  -- Skip "...".
   return ast_node
end

local function parse_table(state)
   local ast_node = new_ast_node(state, "Table")
   skip_token(state)  -- Skip "{"
   local is_inside_parentheses = false

   repeat
      if state.token == "}" then
         break
      else
         local lhs, rhs
         local item_location = location(state)

         if state.token == "TK_NAME" then
            local name = state.token_value
            skip_token(state)  -- Skip name.

            if test_and_skip_token(state, "=") then
               -- `name` = `expr`.
               lhs = init_ast_node({name}, item_location, "String")
               rhs, is_inside_parentheses = parse_expression(state)
            else
               -- `name` is beginning of an expression in array part.
               -- Backtrack lexer to before name.
               state.lexer.line = item_location.line
               state.lexer.line_offset = item_location.offset-item_location.column+1
               state.lexer.offset = item_location.offset
               skip_token(state)  -- Load name again.
               rhs, is_inside_parentheses = parse_expression(state)
            end
         elseif test_and_skip_token(state, "[") then
            -- [ `expr` ] = `expr`.
            lhs = parse_expression(state)
            check_and_skip_token(state, "]")
            check_and_skip_token(state, "=")
            rhs = parse_expression(state)
         else
            -- Expression in array part.
            rhs, is_inside_parentheses = parse_expression(state)
         end

         if lhs then
            -- Pair.
            ast_node[#ast_node+1] = init_ast_node({lhs, rhs}, item_location, "Pair")
         else
            -- Array part item.
            ast_node[#ast_node+1] = rhs
         end
      end
   until not (test_and_skip_token(state, ",") or test_and_skip_token(state, ";"))

   check_and_skip_token(state, "}")
   opt_add_parens(ast_node, is_inside_parentheses)
   return ast_node
end

-- Parses argument list and the statements.
local function parse_function(state, location)
   check_and_skip_token(state, "(")
   local args = {}

   if state.token ~= ")" then  -- Are there arguments?
      repeat
         if state.token == "TK_NAME" then
            args[#args+1] = parse_var(state)
         elseif state.token == "TK_DOTS" then
            args[#args+1] = parse_dots(state)
            break
         else
            error({})
         end
      until not test_and_skip_token(state, ",")
   end

   check_and_skip_token(state, ")")
   local body = parse_block(state)
   check_and_skip_token(state, "TK_END")
   return init_ast_node({args, body}, location, "Function")
end

{'fraction_non_alphanumeric': 0.09823765485953585, 'fraction_numerical': 0.0017448961786773686, 'mean_word_length': 2.730231044581842, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5276034', 'n_tokens_mistral': 3671, 'n_tokens_neox': 3623, 'n_words': 894}