<commit_before>from parsy import string, regex, chain
import re
import pdb
import readline

whitespace = regex(r'\s+', re.MULTILINE)
comment = regex(r';.*')
ignore = (whitespace | comment).many()

lexeme = lambda p: p.skip(ignore)

lparen = lexeme(string('('))
rparen = lexeme(string(')'))
number = lexeme(regex(r'\d+')).map(int)
symbol = lexeme(regex(r'[\d\w_-]+'))
true   = lexeme(string('#t')).result(True)
false  = lexeme(string('#f')).result(False)

atom = true | false | number | symbol

@chain
def form():
    yield lparen
    els = yield expr.many()
    yield rparen
    return els

@chain
def quote():
    yield string("'")
    e = yield expr
    return ['quote', e]

expr = form | quote | atom

program = ignore.then(expr.many())

def test_form():
    result = program.parse('(1 2 3)')
    assert result == [[1, 2, 3]]

def test_quote():
    result = program.parse("'foo '(bar baz)")
    assert result == [['quote', 'foo'], ['quote', ['bar', 'baz']]]

def test_boolean():
    result = program.parse('#t #f')
    assert result == [True, False]

def test_comments():
    result = program.parse(
      """
      ; a program with a comment
      (           foo ; that's a foo
      bar )
      ; some comments at the end
      """
    )

    assert result == [['foo', 'bar']]
<commit_msg>use << and >> in the tests<commit_after>from parsy import string, regex, chain
import re
import pdb
import readline

whitespace = regex(r'\s+', re.MULTILINE)
comment = regex(r';.*')
ignore = (whitespace | comment).many()

lexeme = lambda p: p << ignore

lparen = lexeme(string('('))
rparen = lexeme(string(')'))
number = lexeme(regex(r'\d+')).map(int)
symbol = lexeme(regex(r'[\d\w_-]+'))
true   = lexeme(string('#t')).result(True)
false  = lexeme(string('#f')).result(False)

atom = true | false | number | symbol

@chain
def form():
    yield lparen
    els = yield expr.many()
    yield rparen
    return els

@chain
def quote():
    yield string("'")
    e = yield expr
    return ['quote', e]

expr = form | quote | atom

program = ignore >> expr.many()

def test_form():
    result = program.parse('(1 2 3)')
    assert result == [[1, 2, 3]]

def test_quote():
    result = program.parse("'foo '(bar baz)")
    assert result == [['quote', 'foo'], ['quote', ['bar', 'baz']]]

def test_boolean():
    result = program.parse('#t #f')
    assert result == [True, False]

def test_comments():
    result = program.parse(
      """
      ; a program with a comment
      (           foo ; that's a foo
      bar )
      ; some comments at the end
      """
    )

    assert result == [['foo', 'bar']]

{'fraction_non_alphanumeric': 0.18251829033500191, 'fraction_numerical': 0.004620716211012707, 'mean_word_length': 3.1568, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3469393', 'n_tokens_mistral': 904, 'n_tokens_neox': 891, 'n_words': 258}