<commit_before>/* ************************************************************************* */
/* This file is part of Shard.                                               */
/*                                                                           */
/* Shard is free software: you can redistribute it and/or modify             */
/* it under the terms of the GNU Affero General Public License as            */
/* published by the Free Software Foundation.                                */
/*                                                                           */
/* This program is distributed in the hope that it will be useful,           */
/* but WITHOUT ANY WARRANTY; without even the implied warranty of            */
/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the              */
/* GNU Affero General Public License for more details.                       */
/*                                                                           */
/* You should have received a copy of the GNU Affero General Public License  */
/* along with this program. If not, see <http://www.gnu.org/licenses/>.      */
/* ************************************************************************* */

#include "shard/tokenizer/Tokenizer.hpp"

/* ************************************************************************* */

// C++
#include <algorithm>
#include <cmath>
#include <iostream>

// Shard
#include "shard/StaticArray.hpp"

/* ************************************************************************* */

namespace shard {
inline namespace v1 {
namespace tokenizer {

/* ************************************************************************* */

namespace
{
    static const StaticArray<std::pair<String, KeywordType>, KeywordTypeCount> g_keywordMap
    {{
    #define KEYWORD(name, str) { str, KeywordType::name },
    #include "shard/tokenizer/Token.def"
    }};

    /**
     * @brief returns escaped char value.
     */
    static char getEscaped(const char value)
    {
        switch (value)
        {
            case '"': return '"';
            case '\'': return '\'';
            case '?': return '\?';
            case '\\': return '\\';
            case '0': return '\0';
            case 'a': return '\a';
            case 'b': return '\b';
            case 'f': return '\f';
            case 'n': return '\n';
            case 'r': return '\r';
            case 't': return '\t';
            case 'v': return '\v';
            default: throw InvalidEscapeSequenceException();
        }
    }
}

/* ************************************************************************* */

void Tokenizer::tokenizeNumber()
{
    bool floatFlag = false;
    int digitSize = 0;
    
    const auto readNumber = [&]() -> Token::IntType
    {
        digitSize = 0;
        if (!isDigit())
        {
            throw ExpectedNumberException();
        }
        Token::IntType res = 0;
        do
        {
            res = res * 10 + (m_src.extract() - '0');
            ++digitSize;
        }
        while (isDigit());
        return res;
    };

    Token::IntType base = readNumber();
    
    Token::FloatType decimal = 0;
    if (match('.'))
    {
        floatFlag = true;
        decimal = readNumber();
        decimal = decimal / std::pow(10.0, digitSize);
    }

    Token::FloatType exp = 1;
    if (match('e', 'E'))
    {
        floatFlag = true;
        bool negativeExp = false;
        switch (m_src.get())
        {
            case '-': negativeExp = true;
            case '+': m_src.extract();
            default: break;
        }
        exp = std::pow(10.0, readNumber());
        exp = negativeExp ? (1 / exp) : exp;
    }

    Token::FloatType value = (static_cast<Token::FloatType>(base) + decimal) * exp;

    m_current = floatFlag ? Token(value) : Token(base);
}

void Tokenizer::tokenizeString()
{
    String value;

    while (!match(string_literal_border))
    {
        if (empty())
        {
            throw StringWithoutEndException();
        }
        if (match('\\'))
        {
            if (empty())
            {
                throw StringWithoutEndException();
            }
            value += getEscaped(m_src.extract());
        }
        else
        {
            value += m_src.extract();
        }
    }
    
    m_current = Token(TokenType::String, value);
}

void Tokenizer::tokenizeChar()
{
    Token::CharType value = m_src.extract();
    if (value < 0 || value > 127)
    {
        char bytes;
        if (value <= -112 || value >= 240)
        {
            bytes = 3;
        }
        else if (value <= -96 || value >= 224)
        {
            bytes = 2;
        }
        else if (value <= -64 || value >= 192)
        {
            bytes = 1;
        }
        for (char i = 0; i < bytes; ++i)
        {
            value = (value << 8)|(m_src.extract());
        }
    }<commit_msg>Updated code for more readability<commit_after>/* ************************************************************************* */
/* This file is part of Shard.                                               */
/*                                                                           */
/* Shard is free software: you can redistribute it and/or modify             */
/* it under the terms of the GNU Affero General Public License as            */
/* published by the Free Software Foundation.                                */
/*                                                                           */
/* This program is distributed in the hope that it will be useful,           */
/* but WITHOUT ANY WARRANTY; without even the implied warranty of            */
/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the              */
/* GNU Affero General Public License for more details.                       */
/*                                                                           */
/* You should have received a copy of the GNU Affero General Public License  */
/* along with this program. If not, see <http://www.gnu.org/licenses/>.      */
/* ************************************************************************* */

#include "shard/tokenizer/Tokenizer.hpp"

/* ************************************************************************* */

// C++
#include <algorithm>
#include <cmath>
#include <iostream>

// Shard
#include "shard/StaticArray.hpp"

/* ************************************************************************* */

namespace shard {
inline namespace v1 {
namespace tokenizer {

/* ************************************************************************* */

namespace
{
    static const StaticArray<std::pair<String, KeywordType>, KeywordTypeCount> g_keywordMap
    {{
    #define KEYWORD(name, str) { str, KeywordType::name },
    #include "shard/tokenizer/Token.def"
    }};

    /**
     * @brief returns escaped char value.
     */
    static char getEscaped(const char value)
    {
        switch (value)
        {
            case '"': return '"';
            case '\'': return '\'';
            case '?': return '\?';
            case '\\': return '\\';
            case '0': return '\0';
            case 'a': return '\a';
            case 'b': return '\b';
            case 'f': return '\f';
            case 'n': return '\n';
            case 'r': return '\r';
            case 't': return '\t';
            case 'v': return '\v';
            default: throw InvalidEscapeSequenceException();
        }
    }
}

/* ************************************************************************* */

void Tokenizer::tokenizeNumber()
{
    bool floatFlag = false;
    int digitSize = 0;
    
    const auto readNumber = [&]() -> Token::IntType
    {
        digitSize = 0;
        if (!isDigit())
        {
            throw ExpectedNumberException();
        }
        Token::IntType res = 0;
        do
        {
            res = res * 10 + (m_src.extract() - '0');
            ++digitSize;
        }
        while (isDigit());
        return res;
    };

    Token::IntType base = readNumber();
    
    Token::FloatType decimal = 0;
    if (match('.'))
    {
        floatFlag = true;
        decimal = readNumber();
        decimal = decimal / std::pow(10.0, digitSize);
    }

    Token::FloatType exp = 1;
    if (match('e', 'E'))
    {
        floatFlag = true;
        bool negativeExp = false;
        switch (m_src.get())
        {
            case '-': negativeExp = true;
            case '+': m_src.extract();
            default: break;
        }
        exp = std::pow(10.0, readNumber());
        exp = negativeExp ? (1 / exp) : exp;
    }

    Token::FloatType value = (static_cast<Token::FloatType>(base) + decimal) * exp;

    m_current = floatFlag ? Token(value) : Token(base);
}

void Tokenizer::tokenizeString()
{
    String value;

    while (!match(string_literal_border))
    {
        if (empty())
        {
            throw StringWithoutEndException();
        }
        if (match('\\'))
        {
            if (empty())
            {
                throw StringWithoutEndException();
            }
            value += getEscaped(m_src.extract());
        }
        else
        {
            value += m_src.extract();
        }
    }
    
    m_current = Token(TokenType::String, value);
}

void Tokenizer::tokenizeChar()
{
    Token::CharType value = m_src.extract();
    if (value < 0 || value > 127)
    {
        char additionalBytes;
        if (value >= 0b1111'0000)
        {
            additionalBytes = 3;
        }
        else if (value >= 0b1110'0000)
        {
            additionalBytes = 2;
        }
        else if (value >= 0b1100'0000)
        {
            additionalBytes = 1;
        }
        for (char i = 0; i < additionalBytes; ++i)
        {
            value = (value << 8)|(m_src.extract());
        }
    }
{'fraction_non_alphanumeric': 0.20986016127385934, 'fraction_numerical': 0.009900990099009901, 'mean_word_length': 1.6030818278427206, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1977063', 'n_tokens_mistral': 2582, 'n_tokens_neox': 2437, 'n_words': 672}