<commit_before>"""
"""

from crocs.lex import Lexer, LexMap, LexNode, LexRef, LexChain
from crocs.token import Token

class ListTokens:
    lexmap = LexMap()
    LexChain(lexmap, 
    LexNode('\[', Token),
    LexRef(lexmap),
    # LexNode(',', Token),
    LexNode('\]', Token))

    LexChain(lexmap, LexNode('[0-9]+', Token))
    LexChain(lexmap, LexNode(' +', Token))

# print('Example 1')
# lex = Lexer(ListTokens.lexmap)
# data = '[]'
# lex.feed(data)
# tokens = lex.run()
# print('Consumed:', list(tokens))

# print('Example 2')
# lex = Lexer(ListTokens.lexmap)
# data = '[1, [2, [1,]]]'
# lex.feed(data)
# tokens = lex.run()
# print('Consumed:', list(tokens))
# 
# print('Example 3')
# lex = Lexer(ListTokens.lexmap)
# data = '[1, ss]'
# lex.feed(data)
# tokens = lex.run()
# print('Consumed:', list(tokens))
# 
# print('Example 4')
# lex = Lexer(ListTokens.lexmap)
# data = '[22,2, [, ]]'
# lex.feed(data)
# tokens = lex.run()
# print('Consumed:', list(tokens))

print('Example 5')
lex = Lexer(ListTokens.lexmap)
data = '[1 2]'
lex.feed(data)
tokens = lex.run()
print('Consumed:', list(tokens))


<commit_msg>Adding examples.<commit_after>"""
"""

from crocs.lex import Lexer, LexMap, LexNode, LexRef, LexChain
from crocs.token import Token

class ListTokens:
    lexmap = LexMap()
    LexChain(lexmap, 
    LexNode('\[', Token),
    LexRef(lexmap),
    # LexNode(',', Token),
    LexNode('\]', Token))

    LexChain(lexmap, LexNode('[0-9]+', Token))
    LexChain(lexmap, LexNode(' +', Token))

print('Example 1')
lex = Lexer(ListTokens.lexmap)
data = '[]'
lex.feed(data)
tokens = lex.run()
print('Consumed:', list(tokens))

print('Example 2')
lex = Lexer(ListTokens.lexmap)
data = '[1 [1 2 [1 2 [ 2] 3] 4] ]'
lex.feed(data)
tokens = lex.run()
print('Consumed:', list(tokens))


print('Example 3')
lex = Lexer(ListTokens.lexmap)
data = '[1 [] 3 []]'
lex.feed(data)
tokens = lex.run()
print('Consumed:', list(tokens))

# Should fail.
print('Example 4')
lex = Lexer(ListTokens.lexmap)
data = '[1 [[ 3 []]'
lex.feed(data)
tokens = lex.run()
print('Consumed:', list(tokens))

{'fraction_non_alphanumeric': 0.21462945139557266, 'fraction_numerical': 0.016361886429258902, 'mean_word_length': 4.60377358490566, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6842398', 'n_tokens_mistral': 841, 'n_tokens_neox': 801, 'n_words': 175}