<commit_before>"""
Create pyparsing grammar for problem configuration and options.
"""
from pyparsing import (Word, Group, Suppress, Combine, Optional,
                       Forward, Empty, quotedString, oneOf, removeQuotes,
                       delimitedList, nums, alphas, alphas8bit)

def create_bnf():
    cvt_int = lambda toks: int(toks[0])
    cvt_real = lambda toks: float(toks[0])
    cvt_tuple = lambda toks : tuple(toks.asList())
    cvt_dict = lambda toks: dict(toks.asList())

    # define punctuation as suppressed literals
    (lparen, rparen, lbrack, rbrack,
     lbrace, rbrace, colon) = map(Suppress,"()[]{}:")

    integer = Combine(Optional(oneOf("+ -")) + Word(nums)).setName("integer")
    integer.setParseAction(cvt_int)

    real = Combine(Optional(oneOf("+ -"))
                   + Word(nums) + "." + Optional(Word(nums))).setName("real")
    real.setParseAction(cvt_real)

    tuple_str = Forward()
    list_str = Forward()
    dict_str = Forward()

    list_item = (real | integer | Group(list_str) | tuple_str | dict_str
                 | quotedString.setParseAction(removeQuotes)
                 | Word(alphas8bit + alphas + "_"))
    list_item2 = list_item | Empty().setParseAction(lambda: [None])

    tuple_str << (Suppress("(") + Optional(delimitedList(list_item)) +
                  Optional(Suppress(",")) + Suppress(")"))
    tuple_str.setParseAction(cvt_tuple)

    list_str << (lbrack + Optional(delimitedList(list_item) +
                                   Optional(Suppress(","))) + rbrack)

    dict_entry = Group(list_item + colon + list_item2)
    dict_inner = delimitedList(dict_entry) + Optional(Suppress(","))
    dict_inner.setParseAction(cvt_dict)

    dict_str << (lbrace + Optional(dict_inner) + rbrace)

    return dict_inner
<commit_msg>fix create_bnf() for numbers in identifiers<commit_after>"""
Create pyparsing grammar for problem configuration and options.
"""
from pyparsing import (Word, Group, Suppress, Combine, Optional,
                       Forward, Empty, quotedString, oneOf, removeQuotes,
                       delimitedList, nums, alphas, alphas8bit, alphanums)

def create_bnf():
    cvt_int = lambda toks: int(toks[0])
    cvt_real = lambda toks: float(toks[0])
    cvt_tuple = lambda toks : tuple(toks.asList())
    cvt_dict = lambda toks: dict(toks.asList())

    # define punctuation as suppressed literals
    (lparen, rparen, lbrack, rbrack,
     lbrace, rbrace, colon) = map(Suppress,"()[]{}:")

    integer = Combine(Optional(oneOf("+ -")) + Word(nums)).setName("integer")
    integer.setParseAction(cvt_int)

    real = Combine(Optional(oneOf("+ -"))
                   + Word(nums) + "." + Optional(Word(nums))).setName("real")
    real.setParseAction(cvt_real)

    tuple_str = Forward()
    list_str = Forward()
    dict_str = Forward()

    list_item = (real | integer | Group(list_str) | tuple_str | dict_str
                 | quotedString.setParseAction(removeQuotes)
                 | Word(alphas8bit + alphas, alphas8bit + alphanums + "_"))
    list_item2 = list_item | Empty().setParseAction(lambda: [None])

    tuple_str << (Suppress("(") + Optional(delimitedList(list_item)) +
                  Optional(Suppress(",")) + Suppress(")"))
    tuple_str.setParseAction(cvt_tuple)

    list_str << (lbrack + Optional(delimitedList(list_item) +
                                   Optional(Suppress(","))) + rbrack)

    dict_entry = Group(list_item + colon + list_item2)
    dict_inner = delimitedList(dict_entry) + Optional(Suppress(","))
    dict_inner.setParseAction(cvt_dict)

    dict_str << (lbrace + Optional(dict_inner) + rbrace)

    return dict_inner

{'fraction_non_alphanumeric': 0.13488372093023257, 'fraction_numerical': 0.0035567715458276333, 'mean_word_length': 3.183066361556064, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1326093', 'n_tokens_mistral': 1210, 'n_tokens_neox': 1151, 'n_words': 228}