<commit_before>"""Package provides template tags manager and base tags list 
"""

def parse_token(token):
    tokens = []
    delim = None
    sentence = None
    for word in [word for word in token.split(' ') if len(word) > 0]:
        if delim is None:
            idx = word.find('"')
            if idx < 0:
                idx = word.find("'")
            if idx >= 0:
                delim = word[idx]
                if idx > 0:
                    tokens.append(word[0:idx])
                word = word[idx+1:]
                if delim in word:
                    parts = word.split(delim)
                    tokens.append(parts[0])
                    if len(parts) > 1 and len(parts[1]) > 0:
                        tokens.append(parts[1])
                    delim = None
                else:
                    sentence = [word]
            else: 
                tokens.append(word)
        else:
            if delim in word:
                parts = word.split(delim)
                sentence.append(parts[0])
                tokens.append(" ".join(sentence))
                if len(parts) > 1 and len(parts[1]) > 0:
                    tokens.append(parts[1])
                delim = None
            else:
                sentence.append(word)
    return tokens


class TagManager(object):
    """Class used for tags manipulation 
    """
<commit_msg>Token parsing now saves the token types<commit_after>"""Package provides template tags manager and base tags list 
"""
VARIABLE    = 0
STRING      = 1
NUMBER      = 2

def parse_token(token):
    tokens = []
    token_types = []
    delim = None
    sentence = None
    for word in [word for word in token.split(' ') if len(word) > 0]:
        if delim is None:
            idx = word.find('"')
            if idx < 0:
                idx = word.find("'")
            if idx >= 0:
                delim = word[idx]
                if idx > 0:
                    token_types.append()
                    tokens.append(word[0:idx])
                word = word[idx+1:]
                if delim in word:
                    parts = word.split(delim)
                    tokens.append(parts[0])
                    if len(parts) > 1 and len(parts[1]) > 0:
                        token_types.append(STRING)
                        tokens.append(parts[1])
                    delim = None
                else:
                    sentence = [word]
            else:
                tokens.append(word)
                try:
                    float(word)
                    token_types.append(NUMBER)
                except:
                    token_types.append(VARIABLE)
        else:
            if delim in word:
                parts = word.split(delim)
                sentence.append(parts[0])
                tokens.append(" ".join(sentence))
                if len(parts) > 1 and len(parts[1]) > 0:
                    token_types.append(STRING)
                    tokens.append(parts[1])
                delim = None
            else:
                sentence.append(word)
    return tokens, token_types


class TagManager(object):
    """Class used for tags manipulation 
    """

{'fraction_non_alphanumeric': 0.09933565327428029, 'fraction_numerical': 0.011072445428661816, 'mean_word_length': 1.1524846834581348, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3675371', 'n_tokens_mistral': 882, 'n_tokens_neox': 782, 'n_words': 222}