<commit_before>#include <regex>
#include <string>
#include "parser.h"

static ast_type_t identifier_type(const std::string &s)
{
    static const std::regex float_re("^-?(\\d+\\.\\d*|\\d*\\.\\d+)$");
    static const std::regex int_re("^-?\\d+$");
    if(std::regex_match(s, int_re))
        return ast_type_t::INT;
    if(std::regex_match(s, float_re))
        return ast_type_t::FLOAT;
    return ast_type_t::NAME;
}

std::shared_ptr<ASTNode> readObject(std::istream &is)
{
    auto o = std::make_shared<ASTNode>();
    int c;
    c = is.get();
    while(isspace(c) && c != EOF)
        c = is.get();
    if(c == EOF)
        throw end_of_input("");
    if(c == ')')
        throw parse_error("Invalid syntax");
    if(c == '"')
    {
        o->type = ast_type_t::STRING;
        c = is.get();
        while(c != EOF && c != '"')
        {
            o->value.push_back(c);
            c = is.get();
        }
        if(c == EOF)
            throw parse_error("Unclosed string literal");
        return o;
    }
    if(c == '(')
    {
        o->type = ast_type_t::LIST;
        c = is.peek();
        while(c != EOF && c != ')')
        {
            o->list.push_back(readObject(is));
            c = is.peek();
            while(c != EOF && isspace(c))
            {
                is.get();
                c = is.peek();
            }
        }
        if(c == EOF)
            throw parse_error("Unclosed list literal");
        is.get();
        return o;
    }
    if(c == '\'')
    {
        auto quote = std::make_shared<ASTNode>();
        quote->type = ast_type_t::NAME;
        quote->value = "quote";
        o->type = ast_type_t::LIST;
        o->list.push_back(quote);
        o->list.push_back(readObject(is));
        return o;
    }
    o->value.push_back(c);
    c = is.peek();
    while(c != EOF && c != '(' && c != ')' && !isspace(c))
    {
        o->value.push_back(c);
        is.get();
        c = is.peek();
    }<commit_msg>Комментарии<commit_after>#include <regex>
#include <string>
#include "parser.h"

static ast_type_t identifier_type(const std::string &s)
{
    static const std::regex float_re("^-?(\\d+\\.\\d*|\\d*\\.\\d+)$");
    static const std::regex int_re("^-?\\d+$");
    if(std::regex_match(s, int_re))
        return ast_type_t::INT;
    if(std::regex_match(s, float_re))
        return ast_type_t::FLOAT;
    return ast_type_t::NAME;
}

static int skip_whitespace(std::istream &is)
{
    int c;
    while(true)
    {
        c = is.peek();
        while(c != EOF && isspace(c))
        {
            is.get();
            c = is.peek();
        }
        if(c == ';')
        {
            while(c != EOF && c != '\n')
            {
                c = is.get();
            }
        }
        else
            return c;
    }
}

std::shared_ptr<ASTNode> readObject(std::istream &is)
{
    auto o = std::make_shared<ASTNode>();
    skip_whitespace(is);
    int c = is.get();
    if(c == EOF)
        throw end_of_input("");
    if(c == ')')
        throw parse_error("Invalid syntax");
    if(c == '"')
    {
        o->type = ast_type_t::STRING;
        c = is.get();
        while(c != EOF && c != '"')
        {
            o->value.push_back(c);
            c = is.get();
        }
        if(c == EOF)
            throw parse_error("Unclosed string literal");
        return o;
    }
    if(c == '(')
    {
        o->type = ast_type_t::LIST;
        skip_whitespace(is);
        c = is.peek();
        while(c != EOF && c != ')')
        {
            o->list.push_back(readObject(is));
            c = skip_whitespace(is);
        }
        if(c == EOF)
            throw parse_error("Unclosed list literal");
        is.get();
        return o;
    }
    if(c == '\'')
    {
        auto quote = std::make_shared<ASTNode>();
        quote->type = ast_type_t::NAME;
        quote->value = "quote";
        o->type = ast_type_t::LIST;
        o->list.push_back(quote);
        o->list.push_back(readObject(is));
        return o;
    }
    o->value.push_back(c);
    c = is.peek();
    while(c != EOF && c != '(' && c != ')' && !isspace(c))
    {
        o->value.push_back(c);
        is.get();
        c = is.peek();
    }
{'fraction_non_alphanumeric': 0.1912960306073649, 'fraction_numerical': 0.0, 'mean_word_length': 1.6814102564102564, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1494046', 'n_tokens_mistral': 1503, 'n_tokens_neox': 1498, 'n_words': 261}