<commit_before>import logging
import sys
import string
import re

try:
    import pyparsing
    from pyparsing import Literal, Word, Group, Combine, Optional, Forward, alphanums, SkipTo, LineEnd, nums, delimitedList 
except ImportError:
    logging.critical("PyParsing has to be installed on the computer")
    sys.exit(-1)

from langtools.utils.accents import proszeky_to_utf

from machine import Machine
from monoid import Monoid
from constants import deep_cases
from control import ConceptControl

def create_machine(name, partitions):
    return Machine(Monoid(proszeky_to_utf(name).decode("utf-8"), partitions),
                   ConceptControl())

class ParserException(Exception):
    pass

class DefinitionParser:
    _str = set([str, unicode])

    lb = "["
    rb = "]"
    lp = "("<commit_msg>Fixed a unicode conversion bug in the definition parser. 	modified:   definition_parser.py<commit_after>import logging
import sys
import string
import re

try:
    import pyparsing
    from pyparsing import Literal, Word, Group, Combine, Optional, Forward, alphanums, SkipTo, LineEnd, nums, delimitedList 
except ImportError:
    logging.critical("PyParsing has to be installed on the computer")
    sys.exit(-1)

from langtools.utils.accents import proszeky_to_utf

from machine import Machine
from monoid import Monoid
from constants import deep_cases
from control import ConceptControl

def create_machine(name, partitions):
    return Machine(Monoid(proszeky_to_utf(name), partitions),
                   ConceptControl())

class ParserException(Exception):
    pass

class DefinitionParser:
    _str = set([str, unicode])

    lb = "["
    rb = "]"
    lp = "("
{'fraction_non_alphanumeric': 0.07959305804907241, 'fraction_numerical': 0.0017953321364452424, 'mean_word_length': 4.192546583850931, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '85709', 'n_tokens_mistral': 504, 'n_tokens_neox': 478, 'n_words': 158}