<commit_before>/**
 * @file
 * Declares the Puppet language grammar.
 * The grammar defines the Puppet language and is responsible for populating a syntax tree.
 */
#pragma once

#include "token_pos.hpp"
#include "../lexer/token_id.hpp"
#include "../ast/syntax_tree.hpp"
#include "../cast.hpp"
#include <boost/spirit/include/qi.hpp>
#include <boost/phoenix.hpp>

namespace puppet { namespace compiler {

    /**
     * Represents the Puppet language grammar.
     * The grammar is responsible for transforming a stream of tokens into a syntax tree.
     * @tparam Lexer The lexer type to use for token definitions.
     */
    template <typename Lexer>
    struct grammar : boost::spirit::qi::grammar<typename Lexer::iterator_type, puppet::ast::syntax_tree()>
    {
        /**
         * The token iterator type of the grammar.
         */
        typedef typename Lexer::iterator_type iterator_type;

        /**
         * Constructs a Puppet language grammar for the given lexer.
         * @param lexer The lexer to use for token definitions.
         * @param interpolation True if the grammar is being used for string interpolation or false if not.
         */
        grammar(Lexer const& lexer, bool interpolation = false) :
            boost::spirit::qi::grammar<iterator_type, puppet::ast::syntax_tree()>(syntax_tree)
        {
            using namespace boost::spirit::qi;
            using namespace puppet::lexer;
            namespace phx = boost::phoenix;
            
            // A syntax tree is a sequence of statements
            // For string interpolation, end at the first '}' token that isn't part of the grammar
            if (interpolation) {
                syntax_tree =
                    (raw_token('{') > statements > token_pos('}')) [_val = phx::construct<ast::syntax_tree>(_1, _2)];
            } else {
                syntax_tree =
                    statements [ _val = phx::construct<ast::syntax_tree>(_1) ];
            }

            // Statements
            // The Puppet language doesn't really have "statements" in a pedantic sense of the word
            // It considers a statement to be "any expression"
            statements =
                -(statement % -raw_token(';')) > -raw_token(';');
            statement =
                (statement_expression > *binary_statement) [ _val = phx::construct<ast::expression>(_1, _2) ];
            statement_expression =
                (
                    resource_expression          |
                    resource_defaults_expression |
                    resource_override_expression |
                    class_definition_expression  |
                    defined_type_expression      |
                    node_definition_expression
                )                         [ _val = phx::construct<ast::catalog_expression>(_1) ]      |
                statement_call_expression [ _val = phx::construct<ast::control_flow_expression>(_1) ] |
                primary_expression        [ _val = _1 ];
            binary_statement =
                (binary_operator > statement_expression) [ _val = phx::construct<ast::binary_expression>(_1, _2) ];

            // Expressions
            // Expressions in Puppet do not include certain resource expressions (resource, defaults, and override expressions)
            // Those expressions are only available as "statements"
            expressions =
                (expression % raw_token(',')) > -raw_token(',');
            expression =
                (primary_expression > *binary_expression) [ _val = phx::construct<ast::expression>(_1, _2) ];

            // Primary expression
            // The order of the subexpressions is important; specifically basic_expression must come after the other subexpressions
            primary_expression =
                (
                    (
                        unary_expression |
                        catalog_expression |
                        control_flow_expression |
                        basic_expression |
                        (raw_token('(') > expression > raw_token(')'))
                    ) [ _a = _1 ] >>
                        -(
                            (+postfix_subexpression) [ _a = phx::construct<ast::postfix_expression>(_a, _1) ]
                         )
                ) [ _val = _a ];

            // Basic expressions
            basic_expression =
                undef     |
                defaulted |
                boolean   |
                number    |
                string    |
                regex     |
                variable  |
                name      |
                bare_word |
                type      |
                array     |
                hash;
            undef =
                token_pos(token_id::keyword_undef) [ _val = phx::construct<ast::undef>(_1) ];
            defaulted =
                token_pos(token_id::keyword_default) [ _val = phx::construct<ast::defaulted>(_1) ];
            boolean =
                token_pos(token_id::keyword_true)  [ _val = phx::construct<ast::boolean>(_1, true) ] |
                token_pos(token_id::keyword_false) [ _val = phx::construct<ast::boolean>(_1, false) ];
            number =
                lexer.number [ _val = phx::construct<ast::number>(_1) ];
            string =
                (lexer.single_quoted_string | lexer.double_quoted_string | lexer.heredoc) [ _val = phx::construct<ast::string>(_1) ];
            regex =
                token(token_id::regex) [ _val = phx::construct<ast::regex>(_1) ];
            variable =
                token(token_id::variable) [ _val = phx::construct<ast::variable>(_1) ];
            name =
                (token(token_id::name) | token(token_id::statement_call)) [ _val = phx::construct<ast::name>(_1) ];
            bare_word =
                token(token_id::bare_word) [ _val = phx::construct<ast::bare_word>(_1) ];
            type =
                token(token_id::type) [ _val = phx::construct<ast::type>(_1) ];
            array =
                ((token_pos('[') | token_pos(token_id::array_start)) > -expressions > raw_token(']')) [ _val = phx::construct<ast::array>(_1, _2) ];
            hash =
                (token_pos('{') > -(hash_pair % raw_token(',')) > -raw_token(',') > raw_token('}')) [ _val = phx::construct<ast::hash>(_1, _2) ];
            hash_pair =
                (expression > raw_token(token_id::fat_arrow) > expression) [ _val = phx::construct<ast::hash_pair>(_1, _2) ];

            // Control-flow expressions
            control_flow_expression =
                // Selector and method call expressions are postfix
                case_expression |
                if_expression |
                unless_expression |
                function_call_expression;
            case_expression =
                (token_pos(token_id::keyword_case) > expression > raw_token('{') > +case_proposition > raw_token('}')) [ _val = phx::construct<ast::case_expression>(_1, _2, _3) ];
            case_proposition =
                (lambda > raw_token(':') > raw_token('{') > statements > raw_token('}'))      [ _val = phx::construct<ast::case_proposition>(_1, _2) ] |
                (expressions > raw_token(':') > raw_token('{') > statements > raw_token('}')) [ _val = phx::construct<ast::case_proposition>(_1, _2) ];
            if_expression =
                (token_pos(token_id::keyword_if) > expression > raw_token('{') > statements > raw_token('}') > *elsif_expression > -else_expression) [ _val = phx::construct<ast::if_expression>(_1, _2, _3, _4, _5) ];
            elsif_expression =
                (token_pos(token_id::keyword_elsif) > expression > raw_token('{') > statements > raw_token('}')) [ _val = phx::construct<ast::elsif_expression>(_1, _2, _3) ];
            else_expression =
                (token_pos(token_id::keyword_else) > raw_token('{') > statements > raw_token('}')) [ _val = phx::construct<ast::else_expression>(_1, _2) ];
            unless_expression =
                (token_pos(token_id::keyword_unless) > expression > raw_token('{') > statements > raw_token('}') > -else_expression) [ _val = phx::construct<ast::unless_expression>(_1, _2, _3, _4) ];
            function_call_expression =
                ((name >> raw_token('(')) > -expressions > raw_token(')') > -lambda) [ _val = phx::construct<ast::function_call_expression>(_1, _2, _3) ];
            statement_call_expression =
                (token(token_id::statement_call) >> !raw_token('(') >> expressions >> -lambda) [ _val = phx::construct<ast::function_call_expression>(phx::construct<ast::name>(_1), _2, _3) ];
            lambda =
                (token_pos('|') > -(parameter % raw_token(',')) > -raw_token(',') > raw_token('|') > raw_token('{') > statements > raw_token('}')) [ _val = phx::construct<ast::lambda>(_1, _2, _3) ];
            parameter =
                (-type_expression >> matches[raw_token('*')] >> variable >> -(raw_token('=') > expression)) [ _val = phx::construct<ast::parameter>(_1, _2, _3, _4) ];

            // Catalog expressions
            catalog_expression =
                // Everything but collection expressions are statement-level only
                collection_expression;
            resource_expression =
                (raw_token('@') > resource_type > raw_token('{') > (resource_body % raw_token(';')) > -raw_token(';') > raw_token('}'))            [ _val = phx::construct<ast::resource_expression>(_1, _2, ast::resource_status::virtualized) ] |
                (raw_token(token_id::atat) > resource_type > raw_token('{') > (resource_body % raw_token(';')) > -raw_token(';') > raw_token('}')) [ _val = phx::construct<ast::resource_expression>(_1, _2, ast::resource_status::exported) ] |
                ((resource_type >> raw_token('{')) >> (resource_body % raw_token(';')) > -raw_token(';') > raw_token('}'))                         [ _val = phx::construct<ast::resource_expression>(_1, _2) ];
            resource_type =
                name                           [ _val = _1 ] |
                token(token_id::keyword_class) [ _val = phx::construct<ast::name>(_1) ];
            resource_body =
                ((expression >> raw_token(':')) > -(attribute_expression % raw_token(',')) > -raw_token(',')) [ _val = phx::construct<ast::resource_body>(_1, _2) ];
            attribute_expression =
                (attribute_name >> attribute_operator >> expression) [ _val = phx::construct<ast::attribute_expression>(_1, _2, _3) ];
            attribute_operator =
                raw_token(token_id::fat_arrow)  [ _val = ast::attribute_operator::assignment ] |
                raw_token(token_id::plus_arrow) [ _val = ast::attribute_operator::append ];
            attribute_name =
                (
                    token(token_id::name)             |
                    token(token_id::statement_call)   |
                    token(token_id::keyword_and)      |
                    token(token_id::keyword_case)     |
                    token(token_id::keyword_class)    |<commit_msg>Improve compiler error message for some catalog expressions.<commit_after>/**
 * @file
 * Declares the Puppet language grammar.
 * The grammar defines the Puppet language and is responsible for populating a syntax tree.
 */
#pragma once

#include "token_pos.hpp"
#include "../lexer/token_id.hpp"
#include "../ast/syntax_tree.hpp"
#include "../cast.hpp"
#include <boost/spirit/include/qi.hpp>
#include <boost/phoenix.hpp>

namespace puppet { namespace compiler {

    /**
     * Represents the Puppet language grammar.
     * The grammar is responsible for transforming a stream of tokens into a syntax tree.
     * @tparam Lexer The lexer type to use for token definitions.
     */
    template <typename Lexer>
    struct grammar : boost::spirit::qi::grammar<typename Lexer::iterator_type, puppet::ast::syntax_tree()>
    {
        /**
         * The token iterator type of the grammar.
         */
        typedef typename Lexer::iterator_type iterator_type;

        /**
         * Constructs a Puppet language grammar for the given lexer.
         * @param lexer The lexer to use for token definitions.
         * @param interpolation True if the grammar is being used for string interpolation or false if not.
         */
        grammar(Lexer const& lexer, bool interpolation = false) :
            boost::spirit::qi::grammar<iterator_type, puppet::ast::syntax_tree()>(syntax_tree)
        {
            using namespace boost::spirit::qi;
            using namespace puppet::lexer;
            namespace phx = boost::phoenix;
            
            // A syntax tree is a sequence of statements
            // For string interpolation, end at the first '}' token that isn't part of the grammar
            if (interpolation) {
                syntax_tree =
                    (raw_token('{') > statements > token_pos('}')) [_val = phx::construct<ast::syntax_tree>(_1, _2)];
            } else {
                syntax_tree =
                    statements [ _val = phx::construct<ast::syntax_tree>(_1) ];
            }

            // Statements
            // The Puppet language doesn't really have "statements" in a pedantic sense of the word
            // It considers a statement to be "any expression"
            statements =
                -(statement % -raw_token(';')) > -raw_token(';');
            statement =
                (statement_expression > *binary_statement) [ _val = phx::construct<ast::expression>(_1, _2) ];
            statement_expression =
                (
                    resource_expression          |
                    resource_defaults_expression |
                    resource_override_expression |
                    class_definition_expression  |
                    defined_type_expression      |
                    node_definition_expression
                )                         [ _val = phx::construct<ast::catalog_expression>(_1) ]      |
                statement_call_expression [ _val = phx::construct<ast::control_flow_expression>(_1) ] |
                primary_expression        [ _val = _1 ];
            binary_statement =
                (binary_operator > statement_expression) [ _val = phx::construct<ast::binary_expression>(_1, _2) ];

            // Expressions
            // Expressions in Puppet do not include certain resource expressions (resource, defaults, and override expressions)
            // Those expressions are only available as "statements"
            expressions =
                (expression % raw_token(',')) > -raw_token(',');
            expression =
                (primary_expression > *binary_expression) [ _val = phx::construct<ast::expression>(_1, _2) ];

            // Primary expression
            // The order of the subexpressions is important; specifically basic_expression must come after the other subexpressions
            primary_expression =
                (
                    (
                        unary_expression |
                        catalog_expression |
                        control_flow_expression |
                        basic_expression |
                        (raw_token('(') > expression > raw_token(')'))
                    ) [ _a = _1 ] >>
                        -(
                            (+postfix_subexpression) [ _a = phx::construct<ast::postfix_expression>(_a, _1) ]
                         )
                ) [ _val = _a ];

            // Basic expressions
            basic_expression =
                undef     |
                defaulted |
                boolean   |
                number    |
                string    |
                regex     |
                variable  |
                name      |
                bare_word |
                type      |
                array     |
                hash;
            undef =
                token_pos(token_id::keyword_undef) [ _val = phx::construct<ast::undef>(_1) ];
            defaulted =
                token_pos(token_id::keyword_default) [ _val = phx::construct<ast::defaulted>(_1) ];
            boolean =
                token_pos(token_id::keyword_true)  [ _val = phx::construct<ast::boolean>(_1, true) ] |
                token_pos(token_id::keyword_false) [ _val = phx::construct<ast::boolean>(_1, false) ];
            number =
                lexer.number [ _val = phx::construct<ast::number>(_1) ];
            string =
                (lexer.single_quoted_string | lexer.double_quoted_string | lexer.heredoc) [ _val = phx::construct<ast::string>(_1) ];
            regex =
                token(token_id::regex) [ _val = phx::construct<ast::regex>(_1) ];
            variable =
                token(token_id::variable) [ _val = phx::construct<ast::variable>(_1) ];
            name =
                (token(token_id::name) | token(token_id::statement_call)) [ _val = phx::construct<ast::name>(_1) ];
            bare_word =
                token(token_id::bare_word) [ _val = phx::construct<ast::bare_word>(_1) ];
            type =
                token(token_id::type) [ _val = phx::construct<ast::type>(_1) ];
            array =
                ((token_pos('[') | token_pos(token_id::array_start)) > -expressions > raw_token(']')) [ _val = phx::construct<ast::array>(_1, _2) ];
            hash =
                (token_pos('{') > -(hash_pair % raw_token(',')) > -raw_token(',') > raw_token('}')) [ _val = phx::construct<ast::hash>(_1, _2) ];
            hash_pair =
                (expression > raw_token(token_id::fat_arrow) > expression) [ _val = phx::construct<ast::hash_pair>(_1, _2) ];

            // Control-flow expressions
            control_flow_expression =
                // Selector and method call expressions are postfix
                case_expression |
                if_expression |
                unless_expression |
                function_call_expression;
            case_expression =
                (token_pos(token_id::keyword_case) > expression > raw_token('{') > +case_proposition > raw_token('}')) [ _val = phx::construct<ast::case_expression>(_1, _2, _3) ];
            case_proposition =
                (lambda > raw_token(':') > raw_token('{') > statements > raw_token('}'))      [ _val = phx::construct<ast::case_proposition>(_1, _2) ] |
                (expressions > raw_token(':') > raw_token('{') > statements > raw_token('}')) [ _val = phx::construct<ast::case_proposition>(_1, _2) ];
            if_expression =
                (token_pos(token_id::keyword_if) > expression > raw_token('{') > statements > raw_token('}') > *elsif_expression > -else_expression) [ _val = phx::construct<ast::if_expression>(_1, _2, _3, _4, _5) ];
            elsif_expression =
                (token_pos(token_id::keyword_elsif) > expression > raw_token('{') > statements > raw_token('}')) [ _val = phx::construct<ast::elsif_expression>(_1, _2, _3) ];
            else_expression =
                (token_pos(token_id::keyword_else) > raw_token('{') > statements > raw_token('}')) [ _val = phx::construct<ast::else_expression>(_1, _2) ];
            unless_expression =
                (token_pos(token_id::keyword_unless) > expression > raw_token('{') > statements > raw_token('}') > -else_expression) [ _val = phx::construct<ast::unless_expression>(_1, _2, _3, _4) ];
            function_call_expression =
                ((name >> raw_token('(')) > -expressions > raw_token(')') > -lambda) [ _val = phx::construct<ast::function_call_expression>(_1, _2, _3) ];
            statement_call_expression =
                (token(token_id::statement_call) >> !raw_token('(') >> expressions >> -lambda) [ _val = phx::construct<ast::function_call_expression>(phx::construct<ast::name>(_1), _2, _3) ];
            lambda =
                (token_pos('|') > -(parameter % raw_token(',')) > -raw_token(',') > raw_token('|') > raw_token('{') > statements > raw_token('}')) [ _val = phx::construct<ast::lambda>(_1, _2, _3) ];
            parameter =
                (-type_expression >> matches[raw_token('*')] >> variable >> -(raw_token('=') > expression)) [ _val = phx::construct<ast::parameter>(_1, _2, _3, _4) ];

            // Catalog expressions
            catalog_expression =
                // Everything but collection expressions are statement-level only
                collection_expression;
            resource_expression =
                (raw_token('@') > resource_type > raw_token('{') > (resource_body % raw_token(';')) > -raw_token(';') > raw_token('}'))            [ _val = phx::construct<ast::resource_expression>(_1, _2, ast::resource_status::virtualized) ] |
                (raw_token(token_id::atat) > resource_type > raw_token('{') > (resource_body % raw_token(';')) > -raw_token(';') > raw_token('}')) [ _val = phx::construct<ast::resource_expression>(_1, _2, ast::resource_status::exported) ] |
                ((resource_type >> raw_token('{')) >> (resource_body % raw_token(';')) > -raw_token(';') > raw_token('}'))                         [ _val = phx::construct<ast::resource_expression>(_1, _2) ];
            resource_type =
                name                           [ _val = _1 ] |
                token(token_id::keyword_class) [ _val = phx::construct<ast::name>(_1) ];
            resource_body =
                ((expression >> raw_token(':')) > -(attribute_expression % raw_token(',')) > -raw_token(',')) [ _val = phx::construct<ast::resource_body>(_1, _2) ];
            attribute_expression =
                (attribute_name > attribute_operator > expression) [ _val = phx::construct<ast::attribute_expression>(_1, _2, _3) ];
            attribute_operator =
                raw_token(token_id::fat_arrow)  [ _val = ast::attribute_operator::assignment ] |
                raw_token(token_id::plus_arrow) [ _val = ast::attribute_operator::append ];
            attribute_name =
                (
                    token(token_id::name)             |
                    token(token_id::statement_call)   |
                    token(token_id::keyword_and)      |
                    token(token_id::keyword_case)     |
                    token(token_id::keyword_class)    |
{'fraction_non_alphanumeric': 0.13152306510640221, 'fraction_numerical': 0.00704829892016446, 'mean_word_length': 2.026664843429509, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 283, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2219742', 'n_tokens_mistral': 6302, 'n_tokens_neox': 5989, 'n_words': 1183}