<commit_before>#include "interpreter.hpp"
#include "numbers.hpp"
#include "token.hpp"
#include "value.hpp"
#include <sstream>

namespace laskin
{
    typedef std::vector<token>::const_iterator token_iterator;

    static value parse_value(token_iterator&, const token_iterator&);
    static std::vector<value> parse_list(token_iterator&, const token_iterator&);
    static signature parse_function_signature(token_iterator&, const token_iterator&);
    static std::vector<token> parse_block(token_iterator&, const token_iterator&);
    static void parse_if(token_iterator&, const token_iterator&, interpreter&, std::deque<value>&);

    namespace internal
    {
        void initialize_bool(interpreter*);
        void initialize_io(interpreter*);
        void initialize_list(interpreter*);
        void initialize_numbers(interpreter*);
        void initialize_program(interpreter*);
        void initialize_stack(interpreter*);
        void initialize_string(interpreter*);
        void initialize_trigonometry(interpreter*);
    }

    interpreter::interpreter() {}

    interpreter::interpreter(const interpreter& that)
        : m_functions(that.m_functions) {}

    void interpreter::initialize()
    {
        using namespace internal;

        initialize_bool(this);
        initialize_io(this);
        initialize_list(this);
        initialize_numbers(this);
        initialize_program(this);
        initialize_stack(this);
        initialize_string(this);
        initialize_trigonometry(this);
    }

    void interpreter::execute(const std::vector<token>& tokens,
                              std::deque<value>& stack)
        throw(script_error, syntax_error)
    {
        const token_iterator end = tokens.end();

        for (token_iterator current = tokens.begin(); current < end;)
        {
            switch (current->type())
            {
                case token::type_lparen:
                case token::type_rparen:
                case token::type_rbrack:
                case token::type_lbrace:
                case token::type_rbrace:
                {
                    std::stringstream ss;

                    ss << "unexpected " << current->type();

                    throw syntax_error(ss.str());
                }

                case token::type_int:
                    try
                    {
                        stack.push_back(string_to_int(current++->data()));
                    }
                    catch (std::length_error& e)
                    {
                        throw syntax_error(e.what());
                    }
                    break;

                case token::type_real:
                    try
                    {
                        stack.push_back(string_to_real(current++->data()));
                    }
                    catch (std::length_error& e)
                    {
                        throw syntax_error(e.what());
                    }
                    break;

                case token::type_string:
                    stack.push_back(current++->data());
                    break;

                case token::type_lbrack:
                    stack.push_back(parse_list(++current, end));
                    break;

                case token::type_word:
                {
                    const std::string& id = current++->data();

                    if (!id.compare("if"))
                    {
                        parse_if(current, end, *this, stack);
                    } else {
                        auto entry = m_functions.find(id);

                        if (entry)
                        {
                            bool found = false;

                            for (auto& function : entry->value)
                            {
                                if (function.signature().test(stack))
                                {
                                    found = true;
                                    function.invoke(*this, stack);
                                    break;
                                }
                            }
                            if (!found)
                            {
                                throw script_error(
                                        "signature of function `"
                                        + id
                                        + "' does not match with given stack"
                                );
                            }
                        } else {
                            throw script_error(
                                    "undefined function `"
                                    + id
                                    + "'"
                            );
                        }
                    }
                    break;
                }

                case token::type_colon:
                {
                    std::string name;
                    class signature signature;
                    std::vector<token> body;

                    if (++current < end && current->is(token::type_word))
                    {
                        name = current++->data();
                    }
                    if (current < end && current->is(token::type_lparen))
                    {
                        signature = parse_function_signature(++current, end);
                    }
                    body = parse_block(current, end);
                    if (name.empty())
                    {
                        stack.push_back(function(signature, body));
                    } else {
                        register_function(name, signature, body);
                    }
                }
            }
        }
    }

    void interpreter::register_function(const std::string& name,
                                        const class signature& signature,
                                        void (*callback)(interpreter&, std::deque<value>&))
    {
        hashmap<std::vector<function> >::entry* e = m_functions.find(name);

        if (e)
        {
            for (auto& f : e->value)
            {
                if (signature == f.signature())
                {
                    f = function(signature, callback);
                    return;
                }
            }
            e->value.push_back(function(signature, callback));
        } else {
            std::vector<function> vector;

            vector.push_back(function(signature, callback));
            m_functions.insert(name, vector);
        }
    }

    void interpreter::register_function(const std::string& name,
                                        const std::string& sig,
                                        void (*callback)(interpreter&, std::deque<value>&))
    {
        register_function(name, signature(sig), callback);
    }

    void interpreter::register_function(const std::string& name,
                                        const class signature& signature,
                                        const std::vector<token>& callback)
    {
        hashmap<std::vector<function> >::entry* e = m_functions.find(name);

        if (e)
        {
            for (auto& f : e->value)
            {
                if (signature == f.signature())
                {
                    f = function(signature, callback);
                    return;
                }
            }
            e->value.push_back(function(signature, callback));
        } else {
            std::vector<function> vector;

            vector.push_back(function(signature, callback));
            m_functions.insert(name, vector);
        }
    }

    interpreter& interpreter::assign(const interpreter& that)
    {
        m_functions = that.m_functions;

        return *this;
    }

    static value parse_value(token_iterator& current,
                             const token_iterator& end)
    {
        const class token& token = *current++;

        switch (token.type())
        {
            case token::type_int:
                try
                {
                    return string_to_int(current++->data());
                }
                catch (std::length_error& e)
                {
                    throw syntax_error(e.what());
                }

            case token::type_real:
                try
                {
                    return string_to_real(current++->data());
                }
                catch (std::length_error& e)
                {
                    throw syntax_error(e.what());
                }

            case token::type_string:
                return token.data();

            case token::type_lbrack:
                return parse_list(current, end);

            case token::type_colon:
            {
                class signature signature;

                if (current < end && current->is(token::type_lparen))
                {
                    signature = parse_function_signature(++current, end);
                }

                return function(signature, parse_block(current, end));
            }

            default:
            {
                std::stringstream ss;

                ss << "unexpected " << token.type() << "; missing value";

                throw syntax_error(ss.str());<commit_msg>fix value parsing routine<commit_after>#include "interpreter.hpp"
#include "numbers.hpp"
#include "token.hpp"
#include "value.hpp"
#include <sstream>

namespace laskin
{
    typedef std::vector<token>::const_iterator token_iterator;

    static value parse_value(token_iterator&, const token_iterator&);
    static std::vector<value> parse_list(token_iterator&, const token_iterator&);
    static signature parse_function_signature(token_iterator&, const token_iterator&);
    static std::vector<token> parse_block(token_iterator&, const token_iterator&);
    static void parse_if(token_iterator&, const token_iterator&, interpreter&, std::deque<value>&);

    namespace internal
    {
        void initialize_bool(interpreter*);
        void initialize_io(interpreter*);
        void initialize_list(interpreter*);
        void initialize_numbers(interpreter*);
        void initialize_program(interpreter*);
        void initialize_stack(interpreter*);
        void initialize_string(interpreter*);
        void initialize_trigonometry(interpreter*);
    }

    interpreter::interpreter() {}

    interpreter::interpreter(const interpreter& that)
        : m_functions(that.m_functions) {}

    void interpreter::initialize()
    {
        using namespace internal;

        initialize_bool(this);
        initialize_io(this);
        initialize_list(this);
        initialize_numbers(this);
        initialize_program(this);
        initialize_stack(this);
        initialize_string(this);
        initialize_trigonometry(this);
    }

    void interpreter::execute(const std::vector<token>& tokens,
                              std::deque<value>& stack)
        throw(script_error, syntax_error)
    {
        const token_iterator end = tokens.end();

        for (token_iterator current = tokens.begin(); current < end;)
        {
            switch (current->type())
            {
                case token::type_lparen:
                case token::type_rparen:
                case token::type_rbrack:
                case token::type_lbrace:
                case token::type_rbrace:
                {
                    std::stringstream ss;

                    ss << "unexpected " << current->type();

                    throw syntax_error(ss.str());
                }

                case token::type_int:
                    try
                    {
                        stack.push_back(string_to_int(current++->data()));
                    }
                    catch (std::length_error& e)
                    {
                        throw syntax_error(e.what());
                    }
                    break;

                case token::type_real:
                    try
                    {
                        stack.push_back(string_to_real(current++->data()));
                    }
                    catch (std::length_error& e)
                    {
                        throw syntax_error(e.what());
                    }
                    break;

                case token::type_string:
                    stack.push_back(current++->data());
                    break;

                case token::type_lbrack:
                    stack.push_back(parse_list(++current, end));
                    break;

                case token::type_word:
                {
                    const std::string& id = current++->data();

                    if (!id.compare("if"))
                    {
                        parse_if(current, end, *this, stack);
                    } else {
                        auto entry = m_functions.find(id);

                        if (entry)
                        {
                            bool found = false;

                            for (auto& function : entry->value)
                            {
                                if (function.signature().test(stack))
                                {
                                    found = true;
                                    function.invoke(*this, stack);
                                    break;
                                }
                            }
                            if (!found)
                            {
                                throw script_error(
                                        "signature of function `"
                                        + id
                                        + "' does not match with given stack"
                                );
                            }
                        } else {
                            throw script_error(
                                    "undefined function `"
                                    + id
                                    + "'"
                            );
                        }
                    }
                    break;
                }

                case token::type_colon:
                {
                    std::string name;
                    class signature signature;
                    std::vector<token> body;

                    if (++current < end && current->is(token::type_word))
                    {
                        name = current++->data();
                    }
                    if (current < end && current->is(token::type_lparen))
                    {
                        signature = parse_function_signature(++current, end);
                    }
                    body = parse_block(current, end);
                    if (name.empty())
                    {
                        stack.push_back(function(signature, body));
                    } else {
                        register_function(name, signature, body);
                    }
                }
            }
        }
    }

    void interpreter::register_function(const std::string& name,
                                        const class signature& signature,
                                        void (*callback)(interpreter&, std::deque<value>&))
    {
        hashmap<std::vector<function> >::entry* e = m_functions.find(name);

        if (e)
        {
            for (auto& f : e->value)
            {
                if (signature == f.signature())
                {
                    f = function(signature, callback);
                    return;
                }
            }
            e->value.push_back(function(signature, callback));
        } else {
            std::vector<function> vector;

            vector.push_back(function(signature, callback));
            m_functions.insert(name, vector);
        }
    }

    void interpreter::register_function(const std::string& name,
                                        const std::string& sig,
                                        void (*callback)(interpreter&, std::deque<value>&))
    {
        register_function(name, signature(sig), callback);
    }

    void interpreter::register_function(const std::string& name,
                                        const class signature& signature,
                                        const std::vector<token>& callback)
    {
        hashmap<std::vector<function> >::entry* e = m_functions.find(name);

        if (e)
        {
            for (auto& f : e->value)
            {
                if (signature == f.signature())
                {
                    f = function(signature, callback);
                    return;
                }
            }
            e->value.push_back(function(signature, callback));
        } else {
            std::vector<function> vector;

            vector.push_back(function(signature, callback));
            m_functions.insert(name, vector);
        }
    }

    interpreter& interpreter::assign(const interpreter& that)
    {
        m_functions = that.m_functions;

        return *this;
    }

    static value parse_value(token_iterator& current,
                             const token_iterator& end)
    {
        const class token& token = *current++;

        switch (token.type())
        {
            case token::type_int:
                try
                {
                    return string_to_int(token.data());
                }
                catch (std::length_error& e)
                {
                    throw syntax_error(e.what());
                }

            case token::type_real:
                try
                {
                    return string_to_real(token.data());
                }
                catch (std::length_error& e)
                {
                    throw syntax_error(e.what());
                }

            case token::type_string:
                return token.data();

            case token::type_lbrack:
                return parse_list(current, end);

            case token::type_colon:
            {
                class signature signature;

                if (current < end && current->is(token::type_lparen))
                {
                    signature = parse_function_signature(++current, end);
                }

                return function(signature, parse_block(current, end));
            }

            default:
            {
                std::stringstream ss;

                ss << "unexpected " << token.type() << "; missing value";

                throw syntax_error(ss.str());
{'fraction_non_alphanumeric': 0.09493534482758621, 'fraction_numerical': 0.0, 'mean_word_length': 1.0729283002010275, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1415289', 'n_tokens_mistral': 4369, 'n_tokens_neox': 4039, 'n_words': 790}