<commit_before>import sys
import shlex

def mySplit(inLine):
    if sys.version_info > (3, 0):  # is there a clean way to to it?
        return shlex.split(inLine.strip())
    else:
        tempArray = shlex.split(inLine.strip().encode('utf-8'))
        newArray = []
        for item in tempArray:
            newArray.append(item.decode('utf-8'))
        return newArray


def mySplit2(string):
    inPhar = False
    splitted = []
    current = ""
    for char in string:
        if char == '"':
            if inPhar:
                inPhar = False
                splitted.append(current)
                current = ""
            else:
                inPhar = True
        elif inPhar:
            current += char

    return splitted<commit_msg>more acurate splitter for enums<commit_after>import sys
import shlex

def mySplit(inLine):
    if sys.version_info > (3, 0):  # is there a clean way to to it?
        return shlex.split(inLine.strip())
    else:
        tempArray = shlex.split(inLine.strip().encode('utf-8'))
        newArray = []
        for item in tempArray:
            newArray.append(item.decode('utf-8'))
        return newArray


def mySplit2(string):
    inPhar = False
    splitted = []
    current = ""
    for char in string:
        if char == '"':
            if inPhar:
                inPhar = False
            else:
                inPhar = True
        if inPhar is False and char == ",":
            splitted.append(current)
            current = ""
        else:
            current += char

    splitted.append(current)
    return splitted
{'fraction_non_alphanumeric': 0.10316455696202531, 'fraction_numerical': 0.006329113924050633, 'mean_word_length': 1.6796610169491526, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5073405', 'n_tokens_mistral': 474, 'n_tokens_neox': 460, 'n_words': 125}