<commit_before>-- C99 grammar written in lpeg.re.
-- Adapted and translated from plain LPeg grammar for C99
-- written by <NAME> https://github.com/Flymir/ceg
--
-- Copyright (c) 2009 <NAME>
--
-- Permission is hereby granted, free of charge, to any person obtaining a copy
-- of this software and associated documentation files (the "Software"), to deal
-- in the Software without restriction, including without limitation the rights
-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-- copies of the Software, and to permit persons to whom the Software is
-- furnished to do so, subject to the following conditions:
--
-- The above copyright notice and this permission notice shall be included in
-- all copies or substantial portions of the Software.
--
-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-- THE SOFTWARE.

-- Reference used in the original and in this implementation:
-- http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1124.pdf

local c99 = {}

local re = require("relabel")

local defs = {}

local location = require("titan-compiler.location")

c99.tracing = false

defs["trace"] = function(s, i)
    if c99.tracing then
        local line, col = location.get_line_number(s, i)
        print("TRACE", line, col, "[[" ..s:sub(i, i+ 256):gsub("\n.*", "") .. "]]")
    end
    return true
end

local typedefs = {}

local function elem(xs, e)
    for _, x in ipairs(xs) do
        if e == x then
            return true
        end
    end
    return false
end

defs["store_typedef"] = function(s, i, decl)
    if elem(decl.spec, "typedef") then
        if not (decl.ids and decl.ids[1] and decl.ids[1].decl) then
            return false
        end
        local name = decl.ids[1].decl.name or decl.ids[1].decl.declarator.name
        if name then
            typedefs[name] = true
        end
    end
    return true, decl
end

defs["is_typedef"] = function(s, i, id)
    --print("is " .. id .. " a typedef? " .. tostring(not not typedefs[id]))
    return typedefs[id], typedefs[id] and id
end

defs["empty_table"] = function()
    return true, {}
end

--==============================================================================
-- Lexical Rules (used in both preprocessing and language processing)
--==============================================================================

local lexical_rules = [[--lpeg.re

TRACE <- ({} => trace)

EMPTY_TABLE <- ("" => empty_table)

--------------------------------------------------------------------------------
-- Identifiers

IDENTIFIER <- { identifierNondigit (identifierNondigit / [0-9])* } _
identifierNondigit <- [a-zA-Z_]
                    / universalCharacterName<commit_msg>c99: allow registering multiple typedefs in a single statement<commit_after>-- C99 grammar written in lpeg.re.
-- Adapted and translated from plain LPeg grammar for C99
-- written by <NAME> https://github.com/Flymir/ceg
--
-- Copyright (c) 2009 <NAME>
--
-- Permission is hereby granted, free of charge, to any person obtaining a copy
-- of this software and associated documentation files (the "Software"), to deal
-- in the Software without restriction, including without limitation the rights
-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-- copies of the Software, and to permit persons to whom the Software is
-- furnished to do so, subject to the following conditions:
--
-- The above copyright notice and this permission notice shall be included in
-- all copies or substantial portions of the Software.
--
-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-- THE SOFTWARE.

-- Reference used in the original and in this implementation:
-- http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1124.pdf

local c99 = {}

local re = require("relabel")

local defs = {}

local location = require("titan-compiler.location")

c99.tracing = false

defs["trace"] = function(s, i)
    if c99.tracing then
        local line, col = location.get_line_number(s, i)
        print("TRACE", line, col, "[[" ..s:sub(i, i+ 256):gsub("\n.*", "") .. "]]")
    end
    return true
end

local typedefs = {}

local function elem(xs, e)
    for _, x in ipairs(xs) do
        if e == x then
            return true
        end
    end
    return false
end

defs["store_typedef"] = function(s, i, decl)
    if elem(decl.spec, "typedef") then
        if not (decl.ids and decl.ids[1] and decl.ids[1].decl) then
            return false
        end
        for _, id in ipairs(decl.ids) do
            local name = id.decl.name or id.decl.declarator.name
            if name then
                typedefs[name] = true
            end
        end
    end
    return true, decl
end

defs["is_typedef"] = function(s, i, id)
    --print("is " .. id .. " a typedef? " .. tostring(not not typedefs[id]))
    return typedefs[id], typedefs[id] and id
end

defs["empty_table"] = function()
    return true, {}
end

--==============================================================================
-- Lexical Rules (used in both preprocessing and language processing)
--==============================================================================

local lexical_rules = [[--lpeg.re

TRACE <- ({} => trace)

EMPTY_TABLE <- ("" => empty_table)

--------------------------------------------------------------------------------
-- Identifiers

IDENTIFIER <- { identifierNondigit (identifierNondigit / [0-9])* } _
identifierNondigit <- [a-zA-Z_]
                    / universalCharacterName
{'fraction_non_alphanumeric': 0.17932020330368487, 'fraction_numerical': 0.010165184243964422, 'mean_word_length': 2.8233151183970855, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3574666', 'n_tokens_mistral': 1824, 'n_tokens_neox': 1708, 'n_words': 675}