<commit_before>#######################################################################
# Name: textx.py
# Purpose: Implementation of textX language in Arpeggio.
# Author: <NAME> <<EMAIL>>
# Copyright: (c) 2014 <NAME> <<EMAIL>>
# License: MIT License
#
# The idea for this language is shamelessly stolen from the Xtext language
# but there are some subtle differences in both syntax and semantics.
# To make things clear I have named this language textX ;)
#######################################################################

from __future__ import unicode_literals
import sys
if sys.version < '3':
    text = unicode
else:
    text = str

import re
import os
from arpeggio import StrMatch, Optional, ZeroOrMore, OneOrMore, Sequence,\
    OrderedChoice, RegExMatch, Match, NoMatch, EOF, ParsingExpression,\
    SemanticAction, ParserPython, SemanticActionSingleChild
from arpeggio.export import PMDOTExporter
from arpeggio import RegExMatch as _

from .exceptions import TextXSyntaxError, TextXSemanticError
from .const import MULT_ZEROORMORE, MULT_ONEORMORE, \
    MULT_OPTIONAL, RULE_MATCH, RULE_ABSTRACT


# textX grammar
def textx_model():          return ZeroOrMore(import_stm), ZeroOrMore(textx_rule), EOF
# def textx_rule():           return [abstract_rule, match_rule, common_rule,
#                                     mixin_rule, expression_rule]

def import_stm():           return 'import', grammar_to_import
def grammar_to_import():    return _(r'(\w|\.)+')

def textx_rule():           return [abstract_rule, match_rule, common_rule]
# Rules
def common_rule():          return rule_name, Optional(rule_params), ":", common_rule_body, ";"
def match_rule():           return rule_name, Optional(rule_params), ":", match_rule_body, ";"
def abstract_rule():        return rule_name, Optional(rule_params), ":", abstract_rule_body, ";"
def rule_params():          return '[', OneOrMore(rule_param), ']'
def rule_param():           return param_name, Optional('=', string_value)
def param_name():           return ident

def match_rule_body():      return [simple_match, rule_ref], ZeroOrMore("|", [simple_match, rule_ref])
def abstract_rule_body():   return abstract_rule_ref, OneOrMore("|", abstract_rule_ref)
def common_rule_body():     return sequence

def sequence():             return OneOrMore(choice)
def choice():               return repeatable_expr, ZeroOrMore("|", repeatable_expr)
def repeatable_expr():      return expression, Optional(repeat_operator)
def expression():           return [assignment, simple_match, rule_ref, bracketed_sequence]
def bracketed_sequence():   return '(', sequence, ')'
def repeat_operator():      return ['*', '?', '+'], Optional(repeat_modifiers)
def repeat_modifiers():     return '[', OneOrMore([simple_match,
                                                   'eolterm']), ']'
def simple_match():         return [str_match, re_match]

# Assignment
def assignment():           return attribute, assignment_op, assignment_rhs
def attribute():            return ident
def assignment_op():        return ["=", "*=", "+=", "?="]
def assignment_rhs():       return [simple_match, reference], Optional(repeat_modifiers)

# References
def reference():            return [rule_ref, obj_ref]
def rule_ref():             return ident
def abstract_rule_ref():    return ident
def obj_ref():              return '[', class_name, Optional('|', obj_ref_rule), ']'

def rule_name():            return ident
def obj_ref_rule():         return ident
def class_name():           return ident

def str_match():            return [("'", _(r"((\\')|[^'])*"),"'"),\
                                    ('"', _(r'((\\")|[^"])*'),'"')]
def re_match():             return "/", _(r"((\\/)|[^/])*"), "/"
def ident():                return _(r'\w+')
def integer():              return _(r'[-+]?[0-9]+')
def string_value():         return [_(r"'((\\')|[^'])*'"),
                                    _(r'"((\\")|[^"])*"')]

# Comments
def comment():              return [comment_line, comment_block]
def comment_line():         return _(r'//.*?$')
def comment_block():        return _(r'/\*(.|\n)*?\*/')



# Special rules - primitive types
ID          = _(r'[^\d\W]\w*\b', rule_name='ID', root=True)
BOOL        = _(r'(true|false|0|1)\b', rule_name='BOOL', root=True)
INT         = _(r'[-+]?[0-9]+\b', rule_name='INT', root=True)
FLOAT       = _(r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\b', 'FLOAT', root=True)
STRING      = _(r'("[^"]*")|(\'[^\']*\')\b', 'STRING', root=True)
NUMBER      = OrderedChoice(nodes=[FLOAT, INT], rule_name='NUMBER', root=True)
BASETYPE    = OrderedChoice(nodes=[NUMBER, ID, STRING, BOOL],
                            rule_name='BASETYPE', root=True)

BASE_TYPE_RULES = {rule.rule_name: rule
                   for rule in [ID, BOOL, INT, FLOAT,
                                STRING, NUMBER, BASETYPE]}
BASE_TYPE_NAMES = BASE_TYPE_RULES.keys()

for regex in [ID, BOOL, INT, FLOAT, STRING]:
    regex.compile()


def python_type(textx_type_name):
    """Return Python type from the name of base textx type."""
    return {
        'ID': text,<commit_msg>Fix in STRING base type regular expression.<commit_after>#######################################################################
# Name: textx.py
# Purpose: Implementation of textX language in Arpeggio.
# Author: <NAME> <<EMAIL>>
# Copyright: (c) 2014 <NAME> <<EMAIL>ovic AT <EMAIL> DOT <EMAIL>>
# License: MIT License
#
# The idea for this language is shamelessly stolen from the Xtext language
# but there are some subtle differences in both syntax and semantics.
# To make things clear I have named this language textX ;)
#######################################################################

from __future__ import unicode_literals
import sys
if sys.version < '3':
    text = unicode
else:
    text = str

import re
import os
from arpeggio import StrMatch, Optional, ZeroOrMore, OneOrMore, Sequence,\
    OrderedChoice, RegExMatch, Match, NoMatch, EOF, ParsingExpression,\
    SemanticAction, ParserPython, SemanticActionSingleChild
from arpeggio.export import PMDOTExporter
from arpeggio import RegExMatch as _

from .exceptions import TextXSyntaxError, TextXSemanticError
from .const import MULT_ZEROORMORE, MULT_ONEORMORE, \
    MULT_OPTIONAL, RULE_MATCH, RULE_ABSTRACT


# textX grammar
def textx_model():          return ZeroOrMore(import_stm), ZeroOrMore(textx_rule), EOF
# def textx_rule():           return [abstract_rule, match_rule, common_rule,
#                                     mixin_rule, expression_rule]

def import_stm():           return 'import', grammar_to_import
def grammar_to_import():    return _(r'(\w|\.)+')

def textx_rule():           return [abstract_rule, match_rule, common_rule]
# Rules
def common_rule():          return rule_name, Optional(rule_params), ":", common_rule_body, ";"
def match_rule():           return rule_name, Optional(rule_params), ":", match_rule_body, ";"
def abstract_rule():        return rule_name, Optional(rule_params), ":", abstract_rule_body, ";"
def rule_params():          return '[', OneOrMore(rule_param), ']'
def rule_param():           return param_name, Optional('=', string_value)
def param_name():           return ident

def match_rule_body():      return [simple_match, rule_ref], ZeroOrMore("|", [simple_match, rule_ref])
def abstract_rule_body():   return abstract_rule_ref, OneOrMore("|", abstract_rule_ref)
def common_rule_body():     return sequence

def sequence():             return OneOrMore(choice)
def choice():               return repeatable_expr, ZeroOrMore("|", repeatable_expr)
def repeatable_expr():      return expression, Optional(repeat_operator)
def expression():           return [assignment, simple_match, rule_ref, bracketed_sequence]
def bracketed_sequence():   return '(', sequence, ')'
def repeat_operator():      return ['*', '?', '+'], Optional(repeat_modifiers)
def repeat_modifiers():     return '[', OneOrMore([simple_match,
                                                   'eolterm']), ']'
def simple_match():         return [str_match, re_match]

# Assignment
def assignment():           return attribute, assignment_op, assignment_rhs
def attribute():            return ident
def assignment_op():        return ["=", "*=", "+=", "?="]
def assignment_rhs():       return [simple_match, reference], Optional(repeat_modifiers)

# References
def reference():            return [rule_ref, obj_ref]
def rule_ref():             return ident
def abstract_rule_ref():    return ident
def obj_ref():              return '[', class_name, Optional('|', obj_ref_rule), ']'

def rule_name():            return ident
def obj_ref_rule():         return ident
def class_name():           return ident

def str_match():            return [("'", _(r"((\\')|[^'])*"),"'"),\
                                    ('"', _(r'((\\")|[^"])*'),'"')]
def re_match():             return "/", _(r"((\\/)|[^/])*"), "/"
def ident():                return _(r'\w+')
def integer():              return _(r'[-+]?[0-9]+')
def string_value():         return [_(r"'((\\')|[^'])*'"),
                                    _(r'"((\\")|[^"])*"')]

# Comments
def comment():              return [comment_line, comment_block]
def comment_line():         return _(r'//.*?$')
def comment_block():        return _(r'/\*(.|\n)*?\*/')



# Special rules - primitive types
ID          = _(r'[^\d\W]\w*\b', rule_name='ID', root=True)
BOOL        = _(r'(true|false|0|1)\b', rule_name='BOOL', root=True)
INT         = _(r'[-+]?[0-9]+\b', rule_name='INT', root=True)
FLOAT       = _(r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\b', 'FLOAT', root=True)
STRING      = _(r'("[^"]*")|(\'[^\']*\')', 'STRING', root=True)
NUMBER      = OrderedChoice(nodes=[FLOAT, INT], rule_name='NUMBER', root=True)
BASETYPE    = OrderedChoice(nodes=[NUMBER, ID, STRING, BOOL],
                            rule_name='BASETYPE', root=True)

BASE_TYPE_RULES = {rule.rule_name: rule
                   for rule in [ID, BOOL, INT, FLOAT,
                                STRING, NUMBER, BASETYPE]}
BASE_TYPE_NAMES = BASE_TYPE_RULES.keys()

for regex in [ID, BOOL, INT, FLOAT, STRING]:
    regex.compile()


def python_type(textx_type_name):
    """Return Python type from the name of base textx type."""
    return {
        'ID': text,
{'fraction_non_alphanumeric': 0.18013811886003306, 'fraction_numerical': 0.003307071296566482, 'mean_word_length': 3.198448346263781, 'pattern_counts': {'":': 6, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '56395', 'n_tokens_mistral': 3167, 'n_tokens_neox': 3062, 'n_words': 801}