<commit_before>#######################################################################
# Name: textx.py
# Purpose: Implementation of textX language in Arpeggio.
# Author: <NAME> <<EMAIL>>
# Copyright: (c) 2014 <NAME> <<EMAIL>>
# License: MIT License
#
# The idea for this language is shamelessly stolen from the Xtext language
# but there are some subtle differences in both syntax and semantics.
# To make things clear I have named this language textX ;)
#######################################################################

from collections import namedtuple

from arpeggio import StrMatch, Optional, ZeroOrMore, OneOrMore, Sequence,\
    OrderedChoice, RegExMatch, NoMatch, EOF,\
    SemanticAction,ParserPython, Combine, Parser, SemanticActionSingleChild,\
    SemanticActionBodyWithBraces, Terminal
from arpeggio.export import PMDOTExporter, PTDOTExporter
from arpeggio import RegExMatch as _


# textX grammar
def textx_model():          return ZeroOrMore(rule), EOF
def rule():                 return [metaclass, enum]
def enum():                 return enum_kwd, ident, ':', enum_literal,\
                                    ZeroOrMore("|", enum_literal), ';'
def enum_literal():         return ident, '=', str_match
def metaclass():            return metaclass_name, ":", choice, ';'
def metaclass_name():       return ident

def choice():               return sequence, ZeroOrMore("|", sequence)
def sequence():             return OneOrMore(expr)

def expr():                 return [assignment, terminal_match, rule_match,
                                    bracketed_choice],\
                                    Optional(repeat_operator)
def bracketed_choice():     return '(', choice, ')'
def repeat_operator():      return ['*', '?', '+']

# Assignment
def assignment():           return attribute, assignment_op, assignment_rhs
def attribute():            return ident
def assignment_op():        return ["=", "*=", "+=", "?="]
def assignment_rhs():       return [rule_ref, list_match, terminal_match, bracketed_choice]

# Match
def match():                return [terminal_match, list_match, rule_ref]
def terminal_match():       return [str_match, re_match]
def str_match():            return [("'", _(r"((\\')|[^'])*"),"'"),\
                                    ('"', _(r'((\\")|[^"])*'),'"')]
def re_match():             return "/", _(r"((\\/)|[^/])*"), "/"

def list_match():           return "{", rule_ref, Optional(list_separator), '}'
def list_separator():       return terminal_match

# Rule reference
def rule_ref():             return [rule_match, rule_link]
def rule_match():           return ident
def rule_link():            return '[', rule_name, ']'
#def rule_choice():          return rule_name, ZeroOrMore('|', rule_name)
def rule_name():            return ident

def ident():                return _(r'\w+')
def enum_kwd():             return 'enum'

# Comments
def comment():              return [comment_line, comment_block]
def comment_line():         return _(r'//.*$')
def comment_block():        return _(r'/\*(.|\n)*?\*/')


# Special rules - primitive types
ID      = _(r'[^\d\W]\w*\b', rule_name='ID', root=True)
BOOL    = _(r'true|false|0|1', rule_name='BOOL', root=True)
INT     = _(r'[-+]?[0-9]+', rule_name='INT', root=True)
FLOAT   = _(r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', 'FLOAT', root=True)
STRING  = _(r'("[^"]*")|(\'[^\']*\')', 'STRING', root=True)

def convert(value, _type):
    return {
            'BOOL'  : lambda x: bool(x),
            'INT'   : lambda x: int(x),
            'FLOAT' : lambda x: float(x)
            }.get(_type, lambda x: x)(value)


class RuleMatchCrossRef(object):
    """Helper class used for cross reference resolving."""
    def __init__(self, rule_name, position):
        self.rule_name = rule_name
        self.position = position


# TextX Exceptions
class TextXError(Exception):
    pass


class TextXSemanticError(TextXError):
    pass


class TextXSyntaxError(TextXError):
    pass


# TextX semantic actions
class TextXModelSA(SemanticAction):
    def first_pass(self, parser, node, children):

        class TextXLanguageParser(Parser):
            """
            Parser created from textual textX language description.
            Semantic actions for this parser will construct object
            graph representing model on the given language.
            """
            def __init__(self, *args, **kwargs):
                super(TextXLanguageParser, self).__init__(*args, **kwargs)

                # By default first rule is starting rule
                self.parser_model = children[0]
                self.comments_model = parser._peg_rules.get('__comment', None)

                # Stack for metaclass instances
                self._inst_stack = []
                # Dict for cross-ref resolving
                self._instances = {}

                self.debug = parser.debug

            def _parse(self):
                try:
                    return self.parser_model.parse(self)
                except NoMatch as e:
                    raise TextXSyntaxError(str(e))

            def get_model(self):
                return parse_tree_to_objgraph(self, self.parse_tree)


        textx_parser = TextXLanguageParser()

        textx_parser._metaclasses = parser._metaclasses
        textx_parser._peg_rules = parser._peg_rules

        return textx_parser

    def second_pass(self, parser, textx_parser):
        """Cross reference resolving for parser model."""

        resolved_set = set()

        def resolve(node):
            """Recursively resolve peg rule references."""
            if node in resolved_set or not hasattr(node, 'nodes'):
                return

            resolved_set.add(node)

            def _inner_resolve(rule):
                if type(rule) == RuleMatchCrossRef:
                    if rule.rule_name in textx_parser._peg_rules:
                        rule_name = rule.rule_name
                        rule = textx_parser._peg_rules[rule.rule_name]

                        # Cached rule may be crossref also.
                        while type(rule) == RuleMatchCrossRef:
                            rule_name = rule.rule_name
                            rule = _inner_resolve(rule)
                            # Rewrite rule in the rules map
                            textx_parser._peg_rules[rule_name] = rule

                return rule

            for idx, rule in enumerate(node.nodes):
                # If crossref resolve
                if type(rule) == RuleMatchCrossRef:
                    rule = _inner_resolve(rule)
                    node.nodes[idx] = _inner_resolve(rule)

                # If still unresolved raise exception
                if type(rule) == RuleMatchCrossRef:
                    raise TextXSemanticError('Unexisting rule "{}" at position {}.'\
                            .format(rule.rule_name, parser.pos_to_linecol(rule.position)))

                # Depth-First processing
                resolve(rule)

        resolve(textx_parser.parser_model)

        return textx_parser

textx_model.sem = TextXModelSA()

<commit_msg>Reworked cross-refs resolving in textx parser construction.<commit_after>#######################################################################
# Name: textx.py
# Purpose: Implementation of textX language in Arpeggio.
# Author: <NAME> <<EMAIL> dejan<EMAIL>>
# Copyright: (c) 2014 <NAME> <<EMAIL>>
# License: MIT License
#
# The idea for this language is shamelessly stolen from the Xtext language
# but there are some subtle differences in both syntax and semantics.
# To make things clear I have named this language textX ;)
#######################################################################

from collections import namedtuple

from arpeggio import StrMatch, Optional, ZeroOrMore, OneOrMore, Sequence,\
    OrderedChoice, RegExMatch, NoMatch, EOF,\
    SemanticAction,ParserPython, Combine, Parser, SemanticActionSingleChild,\
    SemanticActionBodyWithBraces, Terminal, ParsingExpression
from arpeggio.export import PMDOTExporter, PTDOTExporter
from arpeggio import RegExMatch as _


# textX grammar
def textx_model():          return ZeroOrMore(rule), EOF
def rule():                 return [metaclass, enum]
def enum():                 return enum_kwd, ident, ':', enum_literal,\
                                    ZeroOrMore("|", enum_literal), ';'
def enum_literal():         return ident, '=', str_match
def metaclass():            return metaclass_name, ":", choice, ';'
def metaclass_name():       return ident

def choice():               return sequence, ZeroOrMore("|", sequence)
def sequence():             return OneOrMore(expr)

def expr():                 return [assignment, terminal_match, rule_match,
                                    bracketed_choice],\
                                    Optional(repeat_operator)
def bracketed_choice():     return '(', choice, ')'
def repeat_operator():      return ['*', '?', '+']

# Assignment
def assignment():           return attribute, assignment_op, assignment_rhs
def attribute():            return ident
def assignment_op():        return ["=", "*=", "+=", "?="]
def assignment_rhs():       return [rule_ref, list_match, terminal_match, bracketed_choice]

# Match
def match():                return [terminal_match, list_match, rule_ref]
def terminal_match():       return [str_match, re_match]
def str_match():            return [("'", _(r"((\\')|[^'])*"),"'"),\
                                    ('"', _(r'((\\")|[^"])*'),'"')]
def re_match():             return "/", _(r"((\\/)|[^/])*"), "/"

def list_match():           return "{", rule_ref, Optional(list_separator), '}'
def list_separator():       return terminal_match

# Rule reference
def rule_ref():             return [rule_match, rule_link]
def rule_match():           return ident
def rule_link():            return '[', rule_name, ']'
#def rule_choice():          return rule_name, ZeroOrMore('|', rule_name)
def rule_name():            return ident

def ident():                return _(r'\w+')
def enum_kwd():             return 'enum'

# Comments
def comment():              return [comment_line, comment_block]
def comment_line():         return _(r'//.*$')
def comment_block():        return _(r'/\*(.|\n)*?\*/')


# Special rules - primitive types
ID      = _(r'[^\d\W]\w*\b', rule_name='ID', root=True)
BOOL    = _(r'true|false|0|1', rule_name='BOOL', root=True)
INT     = _(r'[-+]?[0-9]+', rule_name='INT', root=True)
FLOAT   = _(r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', 'FLOAT', root=True)
STRING  = _(r'("[^"]*")|(\'[^\']*\')', 'STRING', root=True)

def convert(value, _type):
    return {
            'BOOL'  : lambda x: bool(x),
            'INT'   : lambda x: int(x),
            'FLOAT' : lambda x: float(x)
            }.get(_type, lambda x: x)(value)


class RuleMatchCrossRef(object):
    """Helper class used for cross reference resolving."""
    def __init__(self, rule_name, position):
        self.rule_name = rule_name
        self.position = position


# TextX Exceptions
class TextXError(Exception):
    pass


class TextXSemanticError(TextXError):
    pass


class TextXSyntaxError(TextXError):
    pass


# TextX semantic actions
class TextXModelSA(SemanticAction):
    def first_pass(self, parser, node, children):

        class TextXLanguageParser(Parser):
            """
            Parser created from textual textX language description.
            Semantic actions for this parser will construct object
            graph representing model on the given language.
            """
            def __init__(self, *args, **kwargs):
                super(TextXLanguageParser, self).__init__(*args, **kwargs)

                # By default first rule is starting rule
                self.parser_model = children[0]
                self.comments_model = parser._peg_rules.get('__comment', None)

                # Stack for metaclass instances
                self._inst_stack = []
                # Dict for cross-ref resolving
                self._instances = {}

                self.debug = parser.debug

            def _parse(self):
                try:
                    return self.parser_model.parse(self)
                except NoMatch as e:
                    raise TextXSyntaxError(str(e))

            def get_model(self):
                return parse_tree_to_objgraph(self, self.parse_tree)


        textx_parser = TextXLanguageParser()

        textx_parser._metaclasses = parser._metaclasses
        textx_parser._peg_rules = parser._peg_rules

        return textx_parser

    def second_pass(self, parser, textx_parser):
        """Cross reference resolving for parser model."""

        if parser.debug:
            print("RESOLVING XTEXT PARSER: second_pass")

        resolved_set = set()

        def resolve(node):
            """Recursively resolve peg rule references."""

            def _inner_resolve(rule):
                if parser.debug:
                    print("Resolving rule: {}".format(rule))

                if type(rule) == RuleMatchCrossRef:
                    rule_name = rule.rule_name
                    if rule_name in textx_parser._peg_rules:
                        rule = textx_parser._peg_rules[rule_name]
                    else:
                        raise TextXSemanticError('Unexisting rule "{}" at position {}.'\
                                .format(rule.rule_name, parser.pos_to_linecol(rule.position)))

                assert isinstance(rule, ParsingExpression), type(rule)
                # Recurse
                for idx, child in enumerate(rule.nodes):
                    if not child in resolved_set:
                        resolved_set.add(rule)
                        rule.nodes[idx] = _inner_resolve(child)

                return rule

            resolved_set.add(node)
            _inner_resolve(node)

        resolve(textx_parser.parser_model)

        return textx_parser

textx_model.sem = TextXModelSA()


{'fraction_non_alphanumeric': 0.1381085138322968, 'fraction_numerical': 0.002144542140253056, 'mean_word_length': 2.2146139705882355, 'pattern_counts': {'":': 2, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4465465', 'n_tokens_mistral': 3961, 'n_tokens_neox': 3734, 'n_words': 997}