<commit_before>/*
 * Part of tivars_lib_cpp
 * (C) 2015-2018 Adrien "Adriweb" Bertrand
 * https://github.com/adriweb/tivars_lib_cpp
 * License: MIT
 */

#include "TypeHandlers.h"
#include "../utils.h"
#include <unordered_map>
#include <iostream>
#include <iterator>
#include <regex>
#include <fstream>

namespace tivars
{
    namespace TH_Tokenized
    {
        std::unordered_map<uint, std::vector<std::string>> tokens_BytesToName;
        std::unordered_map<std::string, uint> tokens_NameToBytes;
        uchar lengthOfLongestTokenName;
        std::vector<uchar> firstByteOfTwoByteTokens;
        const uint16_t squishedASMTokens[] = { 0xBB6D, 0xEF69, 0xEF7B }; // 83+/84+, 84+CSE, CE
    }

    /* TODO: handle TI-Innovator Send( exception for in-strings tokenization (=> not shortest tokens) */
    data_t TH_Tokenized::makeDataFromString(const std::string& str, const options_t& options)
    {
        (void)options;
        data_t data;

        // two bytes reserved for the size. Filled later
        data.push_back(0); data.push_back(0);

        const uchar maxTokSearchLen = std::min((uchar)str.length(), lengthOfLongestTokenName);

        bool isWithinString = false;

        for (uint strCursorPos = 0; strCursorPos < str.length(); strCursorPos++)
        {
            const std::string currChar = str.substr(strCursorPos, 1);
            if (currChar == "\"")
            {
                isWithinString = !isWithinString;
            } else if (currChar == "\n" || currChar == "→")
            {
                isWithinString = false;
            }
            /* isWithinString => minimum token length, otherwise maximal munch */
            for (uint currentLength = isWithinString ? 1 : maxTokSearchLen;
                 isWithinString ? (currentLength <= maxTokSearchLen) : (currentLength > 0);
                 currentLength += (isWithinString ? 1 : -1))
            {
                std::string currentSubString = str.substr(strCursorPos, currentLength);
                if (tokens_NameToBytes.count(currentSubString))
                {
                    uint tokenValue = tokens_NameToBytes[currentSubString];
                    if (tokenValue > 0xFF)
                    {
                        data.push_back((uchar)(tokenValue >> 8));
                    }
                    data.push_back((uchar)(tokenValue & 0xFF));
                    strCursorPos += currentLength - 1;
                    break;
                }
            }
        }

        uint actualDataLen = (uint) (data.size() - 2);
        data[0] = (uchar)(actualDataLen & 0xFF);
        data[1] = (uchar)((actualDataLen >> 8) & 0xFF);
        return data;
    }

    std::string TH_Tokenized::makeStringFromData(const data_t& data, const options_t& options)
    {
        if (data.size() < 2)
        {
            throw std::invalid_argument("Invalid data array. Needs to contain at least 2 bytes (size fields)");
        }

        uint langIdx = (uint)((has_option(options, "lang") && options.at("lang") == LANG_FR) ? LANG_FR : LANG_EN);

        const int howManyBytes = (data[0] & 0xFF) + ((data[1] & 0xFF) << 8);
        if (howManyBytes != (int)data.size() - 2)
        {
            std::cerr << "[Warning] Byte count (" << (data.size() - 2) << ") and size field (" << howManyBytes  << ") mismatch!";
        }

        if (howManyBytes >= 2)
        {
            const uint16_t twoFirstBytes = (uint16_t) ((data[3] & 0xFF) + ((data[2] & 0xFF) << 8));
            if (std::find(std::begin(squishedASMTokens), std::end(squishedASMTokens), twoFirstBytes) != std::end(squishedASMTokens))
            {
                return "[Error] This is a squished ASM program - cannnot preview it!";
            }
        }

        uint errCount = 0;
        std::string str;
        const size_t dataSize = data.size();
        for (uint i = 2; i < (uint)howManyBytes + 2; i++)
        {
            uint currentToken = data[i];
            uint nextToken = (i < dataSize-1) ? data[i+1] : (uint)-1;
            uint bytesKey = currentToken;
            if (is_in_vector(firstByteOfTwoByteTokens, (uchar)currentToken))
            {
                if (nextToken == (uint)-1)<commit_msg>TH_Tokenized: prevent unlikely OOB access<commit_after>/*
 * Part of tivars_lib_cpp
 * (C) 2015-2018 Adrien "Adriweb" Bertrand
 * https://github.com/adriweb/tivars_lib_cpp
 * License: MIT
 */

#include "TypeHandlers.h"
#include "../utils.h"
#include <unordered_map>
#include <iostream>
#include <iterator>
#include <regex>
#include <fstream>

namespace tivars
{
    namespace TH_Tokenized
    {
        std::unordered_map<uint, std::vector<std::string>> tokens_BytesToName;
        std::unordered_map<std::string, uint> tokens_NameToBytes;
        uchar lengthOfLongestTokenName;
        std::vector<uchar> firstByteOfTwoByteTokens;
        const uint16_t squishedASMTokens[] = { 0xBB6D, 0xEF69, 0xEF7B }; // 83+/84+, 84+CSE, CE
    }

    /* TODO: handle TI-Innovator Send( exception for in-strings tokenization (=> not shortest tokens) */
    data_t TH_Tokenized::makeDataFromString(const std::string& str, const options_t& options)
    {
        (void)options;
        data_t data;

        // two bytes reserved for the size. Filled later
        data.push_back(0); data.push_back(0);

        const uchar maxTokSearchLen = std::min((uchar)str.length(), lengthOfLongestTokenName);

        bool isWithinString = false;

        for (uint strCursorPos = 0; strCursorPos < str.length(); strCursorPos++)
        {
            const std::string currChar = str.substr(strCursorPos, 1);
            if (currChar == "\"")
            {
                isWithinString = !isWithinString;
            } else if (currChar == "\n" || currChar == "→")
            {
                isWithinString = false;
            }
            /* isWithinString => minimum token length, otherwise maximal munch */
            for (uint currentLength = isWithinString ? 1 : maxTokSearchLen;
                 isWithinString ? (currentLength <= maxTokSearchLen) : (currentLength > 0);
                 currentLength += (isWithinString ? 1 : -1))
            {
                std::string currentSubString = str.substr(strCursorPos, currentLength);
                if (tokens_NameToBytes.count(currentSubString))
                {
                    uint tokenValue = tokens_NameToBytes[currentSubString];
                    if (tokenValue > 0xFF)
                    {
                        data.push_back((uchar)(tokenValue >> 8));
                    }
                    data.push_back((uchar)(tokenValue & 0xFF));
                    strCursorPos += currentLength - 1;
                    break;
                }
            }
        }

        uint actualDataLen = (uint) (data.size() - 2);
        data[0] = (uchar)(actualDataLen & 0xFF);
        data[1] = (uchar)((actualDataLen >> 8) & 0xFF);
        return data;
    }

    std::string TH_Tokenized::makeStringFromData(const data_t& data, const options_t& options)
    {
        if (data.size() < 2)
        {
            throw std::invalid_argument("Invalid data array. Needs to contain at least 2 bytes (size fields)");
        }

        uint langIdx = (uint)((has_option(options, "lang") && options.at("lang") == LANG_FR) ? LANG_FR : LANG_EN);

        const int howManyBytes = (data[0] & 0xFF) + ((data[1] & 0xFF) << 8);
        if (howManyBytes != (int)data.size() - 2)
        {
            std::cerr << "[Warning] Byte count (" << (data.size() - 2) << ") and size field (" << howManyBytes  << ") mismatch!";
        }

        if (howManyBytes >= 2)
        {
            const uint16_t twoFirstBytes = (uint16_t) ((data[3] & 0xFF) + ((data[2] & 0xFF) << 8));
            if (std::find(std::begin(squishedASMTokens), std::end(squishedASMTokens), twoFirstBytes) != std::end(squishedASMTokens))
            {
                return "[Error] This is a squished ASM program - cannnot preview it!";
            }
        }

        uint errCount = 0;
        std::string str;
        const size_t dataSize = data.size();
        for (uint i = 2; i < (uint)dataSize; i++)
        {
            uint currentToken = data[i];
            uint nextToken = (i < dataSize-1) ? data[i+1] : (uint)-1;
            uint bytesKey = currentToken;
            if (is_in_vector(firstByteOfTwoByteTokens, (uchar)currentToken))
            {
                if (nextToken == (uint)-1)
{'fraction_non_alphanumeric': 0.12827225130890052, 'fraction_numerical': 0.015825797239409804, 'mean_word_length': 2.0776272427682168, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 39, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2211251', 'n_tokens_mistral': 2654, 'n_tokens_neox': 2481, 'n_words': 572}