<commit_before>"""
Create pyparsing grammar for problem configuration and options.
"""
from pyparsing import (Word, Group, Suppress, Combine, Optional,
                       Forward, Empty, quotedString, oneOf, removeQuotes,
                       delimitedList, nums, alphas, alphanums,
                       Keyword, CaselessLiteral)

word_free = Word(alphas + '][@_-/.+**' + alphanums)
word_strict = Word(alphas, alphas + alphanums + '_' )

(lparen, rparen, lbrack, rbrack,
 lbrace, rbrace, colon, equal_sign) = map(Suppress, '()[]{}:=')

integer = Combine(Optional(oneOf('+ -')) + Word(nums)).setName('integer')
cvt_int = lambda toks: int(toks[0])
integer.setParseAction(cvt_int)

boolean_true = Keyword('True', caseless=True)
boolean_true.setParseAction(lambda x: True)
boolean_false = Keyword('False', caseless=True)
boolean_false.setParseAction(lambda x: False)

boolean = boolean_true | boolean_false

none = Keyword('None', caseless=True)

cvt_none = lambda toks: [None]
none.setParseAction(cvt_none)

e = CaselessLiteral("e")
real = (Combine(Optional(oneOf('+ -')) + Word(nums)
               + '.' + Optional(Word(nums))
               + Optional(e + Optional(oneOf('+ -')) + Word(nums)))
        | Combine(Optional(oneOf('+ -')) + Word(nums)
               + Optional('.') + Optional(Word(nums))
               + e + Optional(oneOf('+ -')) + Word(nums))).setName('real')
cvt_real = lambda toks: float(toks[0])
real.setParseAction(cvt_real)

cmplx = real + CaselessLiteral('j')
cvt_cmplx = lambda toks: complex(toks[0])
cmplx.setParseAction(cvt_cmplx)

array_index = integer + Optional(colon + integer
                                 + Optional(colon + integer))
cvt_array_index = lambda toks: int(toks[0]) if len(toks) == 1 \
                  else slice(*toks)
array_index.setParseAction(cvt_array_index)
array_braces = lbrack + array_index + rbrack

def create_bnf(allow_tuple=False, free_word=False):
    word = word_free if free_word else word_strict
    defs = get_standard_type_defs(word)

    if allow_tuple:
        return defs['dict'].inner | defs['tuple'].inner
    else:
        return defs['dict'].inner

def list_of(element, *elements):
    """
    Return lexical element that parses a list of items. The items can be a one
    or several lexical elements. For example, result of ``list_of(real,
    integer)`` parses list of real or integer numbers.
    """
    for e in elements:
        element |= e
    lst = delimitedList(element)
    return lst + Optional(Suppress(','))

def get_standard_type_defs(word=word_free):
    """<commit_msg>Allow empty dict in dict_from_string<commit_after>"""
Create pyparsing grammar for problem configuration and options.
"""
from pyparsing import (Word, Group, Suppress, Combine, Optional,
                       Forward, Empty, quotedString, oneOf, removeQuotes,
                       delimitedList, nums, alphas, alphanums,
                       Keyword, CaselessLiteral)

word_free = Word(alphas + '][@_-/.+**' + alphanums)
word_strict = Word(alphas, alphas + alphanums + '_' )

(lparen, rparen, lbrack, rbrack,
 lbrace, rbrace, colon, equal_sign) = map(Suppress, '()[]{}:=')

integer = Combine(Optional(oneOf('+ -')) + Word(nums)).setName('integer')
cvt_int = lambda toks: int(toks[0])
integer.setParseAction(cvt_int)

boolean_true = Keyword('True', caseless=True)
boolean_true.setParseAction(lambda x: True)
boolean_false = Keyword('False', caseless=True)
boolean_false.setParseAction(lambda x: False)

boolean = boolean_true | boolean_false

none = Keyword('None', caseless=True)

cvt_none = lambda toks: [None]
none.setParseAction(cvt_none)

e = CaselessLiteral("e")
real = (Combine(Optional(oneOf('+ -')) + Word(nums)
               + '.' + Optional(Word(nums))
               + Optional(e + Optional(oneOf('+ -')) + Word(nums)))
        | Combine(Optional(oneOf('+ -')) + Word(nums)
               + Optional('.') + Optional(Word(nums))
               + e + Optional(oneOf('+ -')) + Word(nums))).setName('real')
cvt_real = lambda toks: float(toks[0])
real.setParseAction(cvt_real)

cmplx = real + CaselessLiteral('j')
cvt_cmplx = lambda toks: complex(toks[0])
cmplx.setParseAction(cvt_cmplx)

array_index = integer + Optional(colon + integer
                                 + Optional(colon + integer))
cvt_array_index = lambda toks: int(toks[0]) if len(toks) == 1 \
                  else slice(*toks)
array_index.setParseAction(cvt_array_index)
array_braces = lbrack + array_index + rbrack

def create_bnf(allow_tuple=False, free_word=False):
    word = word_free if free_word else word_strict
    defs = get_standard_type_defs(word)

    if allow_tuple:
        return defs['dict'].inner | Optional(defs['tuple'].inner)
    else:
        return Optional(defs['dict'].inner)

def list_of(element, *elements):
    """
    Return lexical element that parses a list of items. The items can be a one
    or several lexical elements. For example, result of ``list_of(real,
    integer)`` parses list of real or integer numbers.
    """
    for e in elements:
        element |= e
    lst = delimitedList(element)
    return lst + Optional(Suppress(','))

def get_standard_type_defs(word=word_free):
    """
{'fraction_non_alphanumeric': 0.13665315576143602, 'fraction_numerical': 0.0019301293186643506, 'mean_word_length': 3.5899025686448183, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4705677', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1694, 'n_words': 386}