<commit_before># Python Lisp parser

import math
import operator as op
import re
import random
import datetime
from time import time
from collections import OrderedDict
from red_star.rs_errors import CustomCommandSyntaxError, CommandSyntaxError
from red_star.rs_utils import is_positive
from functools import reduce

Symbol = str
Number = (int, float)
List = list

_args = 'args'
_quote = 'quote'
_if = 'if'
_set = ':='
_define = 'define'
_def = 'def'
_lambda = 'lambda'
_begin = 'do'
_unquote = 'unquote'
_access = '>>'
_while = 'while'
_print = 'print'
_try = 'try'
_raise = 'raise'

# temporarily swap a bunch of escaped character into some temporarily values and into un-escaped versions
escapes = (("\\\\", "\uff00", "\\"), ("\\\"", "\uff01", "\""), ("\\n", "\uff02", "\n"), ("\\;", "\uff03", ";"))
# search for lisp instances -
# > comments - ;.*?(?:\n|$) - start with ;, end at line end/end of file, whichever's shorter.
# > strings  - \".*?\"      - start and end with quotes, gets the shortest option.
# > brackets - \(|\)        - get closing and opening bracket symbols as separate tokens.
# > tokens   - [^()\";\S]   - get everything that isn't a special char or whitespace.
tokenizer = re.compile(r";.*?(?:\n|$)|\".*?\"|\(|\)|[^()\";\s]+", re.DOTALL)


class Empty:
    pass


def l_escape(string: str) -> str:
    for p, r, _ in escapes:
        string = string.replace(p, r)
    return string


def l_restore(string: str) -> str:
    for _, p, r in escapes:
        string = string.replace(p, r)
    return string


def tokenize(string: str) -> list:
    return tokenizer.findall(l_escape(string))


def parse(program: str):
    return read_from_tokens(tokenize(program))


def joiner(iterable):
    return reduce(lambda s, i: s + i if i[0] in '()"' or s[-1] in '()"' else s + ' ' + i, iterable)


def reprint(ast: [list, int, str, float])-> str:
    """
    Takes an RSLisp Abstract Syntax Tree and prints out the corresponding lisp code.
    :param ast:
    :return:
    """
    if isinstance(ast, list):
        if len(ast) > 0:
            if ast[0] == 'quote' and isinstance(ast[1], str):
                return f'"{ast[1]}"'
            else:
                return '('+joiner([reprint(x) for x in ast])+')'
        else:
            return "()"
    else:
        return str(ast)


def minify(program: [str, list]):
    """
    Takes a program or an abstract syntax tree and returns the corresponding lisp code, with comments and extra
    whitespace stripped.
    :param program:
    :return:
    """
    return reprint(program if isinstance(program, list) else parse(program))


def read_from_tokens(tokens):
    if len(tokens) == 0:
        raise CustomCommandSyntaxError('unexpected EOF while reading')
    token = tokens.pop(0)
<commit_msg>Bug fix: lisp reprinter now actually escapes special characters.<commit_after># Python Lisp parser

import math
import operator as op
import re
import random
import datetime
from time import time
from collections import OrderedDict
from red_star.rs_errors import CustomCommandSyntaxError, CommandSyntaxError
from red_star.rs_utils import is_positive
from functools import reduce

Symbol = str
Number = (int, float)
List = list

_args = 'args'
_quote = 'quote'
_if = 'if'
_set = ':='
_define = 'define'
_def = 'def'
_lambda = 'lambda'
_begin = 'do'
_unquote = 'unquote'
_access = '>>'
_while = 'while'
_print = 'print'
_try = 'try'
_raise = 'raise'

# temporarily swap a bunch of escaped character into some temporarily values and into un-escaped versions
escapes = (("\\\\", "\uff00", "\\"), ("\\\"", "\uff01", "\""), ("\\n", "\uff02", "\n"), ("\\;", "\uff03", ";"))
# search for lisp instances -
# > comments - ;.*?(?:\n|$) - start with ;, end at line end/end of file, whichever's shorter.
# > strings  - \".*?\"      - start and end with quotes, gets the shortest option.
# > brackets - \(|\)        - get closing and opening bracket symbols as separate tokens.
# > tokens   - [^()\";\S]   - get everything that isn't a special char or whitespace.
tokenizer = re.compile(r";.*?(?:\n|$)|\".*?\"|\(|\)|[^()\";\s]+", re.DOTALL)


class Empty:
    pass


def l_escape(string: str) -> str:
    for p, r, _ in escapes:
        string = string.replace(p, r)
    return string


def l_restore(string: str) -> str:
    for _, p, r in escapes:
        string = string.replace(p, r)
    return string


def l_revert(string: str) -> str:
    for r, _, p in escapes:
        string = string.replace(p, r)
    return string


def tokenize(string: str) -> list:
    return tokenizer.findall(l_escape(string))


def parse(program: str):
    return read_from_tokens(tokenize(program))


def joiner(iterable):
    return reduce(lambda s, i: s + i if i[0] in '()"' or s[-1] in '()"' else s + ' ' + i, iterable)


def reprint(ast: [list, int, str, float])-> str:
    """
    Takes an RSLisp Abstract Syntax Tree and prints out the corresponding lisp code.
    :param ast:
    :return:
    """
    if isinstance(ast, list):
        if len(ast) > 0:
            if ast[0] == 'quote' and isinstance(ast[1], str):
                return f'"{l_revert(ast[1])}"'
            else:
                return '('+joiner([reprint(x) for x in ast])+')'
        else:
            return "()"
    else:
        return l_revert(str(ast))


def minify(program: [str, list]):
    """
    Takes a program or an abstract syntax tree and returns the corresponding lisp code, with comments and extra
    whitespace stripped.
    :param program:
    :return:
    """
    return reprint(program if isinstance(program, list) else parse(program))


def read_from_tokens(tokens):
    if len(tokens) == 0:
        raise CustomCommandSyntaxError('unexpected EOF while reading')
    token = tokens.pop(0)

{'fraction_non_alphanumeric': 0.15236763934999126, 'fraction_numerical': 0.005591473003669404, 'mean_word_length': 3.028149190710767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2616202', 'n_tokens_mistral': 1918, 'n_tokens_neox': 1836, 'n_words': 616}