<commit_before># Python Lisp parser

import math
import operator as op
import re
import random
import datetime
from rs_errors import CustomCommandSyntaxError, CommandSyntaxError
from rs_utils import is_positive

Symbol = str
Number = (int, float)
List = list


_args = 'args'
_quote = 'quote'
_if = 'if'
_set = ':='
_define = 'define'
_def = 'def'
_lambda = 'lambda'
_begin = 'do'
_unquote = 'unquote'
_checkexpect = 'check-expect'
_checkwithin = 'check-within'
_member = 'member?'
_struct = 'struct'
_pass = '<PASSWORD>'
_access = '>>'
_while = 'while'
_print = 'print'

escapes = (("\\\\", "\uff00", "\\"), ("\\\"", "\uff01", "\""), ("\\\n", "\uff02", "\n"))


def l_escape(string: str) -> str:
    for p, r, _ in escapes:
        string = string.replace(p, r)
    return string


def l_restore(string: str) -> str:
    for _, p, r in escapes:
        string = string.replace(p, r)
    return string


def tokenize(string: str) -> list:
    return re.findall(r"(?:\").*?(?:\")|\(|\)|[^()\" ]+", l_escape(string))


def parse(program: str):
    return read_from_tokens(tokenize(program))


def read_from_tokens(tokens):
    if len(tokens) == 0:
        raise CustomCommandSyntaxError('unexpected EOF while reading')
    token = tokens.pop(0)

    if re.match(r'\".*\"', token):
        return ['quote', l_restore(token[1:-1])]

    elif '(' == token:
        l = []
        while tokens[0] != ')':
            t = read_from_tokens(tokens)
            if t != '':
                l.append(t)
        tokens.pop(0)
        return l

    elif ';' == token:
        l = []
        l.append(token)
        while tokens[0] != '\n':
            l.append(read_from_tokens(tokens))
        new_line = tokens.pop(0)
        l.append(new_line)
        string = l_restore(" ".join(l))
        return ['quote', string]

    elif ')' == token:
        raise CustomCommandSyntaxError('unexpected )')
    else:
        return atom(re.sub(r"^\s+|\s+$", "", token))


def atom(token: str):
    try:<commit_msg>Fixed parsing of multi-line strings.<commit_after># Python Lisp parser

import math
import operator as op
import re
import random
import datetime
from rs_errors import CustomCommandSyntaxError, CommandSyntaxError
from rs_utils import is_positive

Symbol = str
Number = (int, float)
List = list


_args = 'args'
_quote = 'quote'
_if = 'if'
_set = ':='
_define = 'define'
_def = 'def'
_lambda = 'lambda'
_begin = 'do'
_unquote = 'unquote'
_checkexpect = 'check-expect'
_checkwithin = 'check-within'
_member = 'member?'
_struct = 'struct'
_pass = '<PASSWORD>'
_access = '>>'
_while = 'while'
_print = 'print'

escapes = (("\\\\", "\uff00", "\\"), ("\\\"", "\uff01", "\""), ("\\n", "\uff02", "\n"))


def l_escape(string: str) -> str:
    for p, r, _ in escapes:
        string = string.replace(p, r)
    return string


def l_restore(string: str) -> str:
    for _, p, r in escapes:
        string = string.replace(p, r)
    return string


def tokenize(string: str) -> list:
    return re.findall(r"(?:\").*?(?:\")|\(|\)|[^()\" ]+", l_escape(string), re.DOTALL)


def parse(program: str):
    return read_from_tokens(tokenize(program))


def read_from_tokens(tokens):
    if len(tokens) == 0:
        raise CustomCommandSyntaxError('unexpected EOF while reading')
    token = tokens.pop(0)

    if re.match(r'\".*\"', token, re.DOTALL):
        return ['quote', l_restore(token[1:-1])]

    elif '(' == token:
        l = []
        while tokens[0] != ')':
            t = read_from_tokens(tokens)
            if t != '':
                l.append(t)
        tokens.pop(0)
        return l

    elif ';' == token:
        l = []
        l.append(token)
        while tokens[0] != '\n':
            l.append(read_from_tokens(tokens))
        new_line = tokens.pop(0)
        l.append(new_line)
        string = l_restore(" ".join(l))
        return ['quote', string]

    elif ')' == token:
        raise CustomCommandSyntaxError('unexpected )')
    else:
        return atom(re.sub(r"^\s+|\s+$", "", token))


def atom(token: str):
    try:
{'fraction_non_alphanumeric': 0.17791563275434244, 'fraction_numerical': 0.006947890818858561, 'mean_word_length': 2.7049632352941178, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5787636', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1372, 'n_words': 345}