<commit_before>#include "command_manager.hh"

#include "utils.hh"
#include "assert.hh"
#include "context.hh"
#include "shell_manager.hh"
#include "register_manager.hh"

#include <algorithm>

namespace Kakoune
{

bool CommandManager::command_defined(const String& command_name) const
{
    return m_commands.find(command_name) != m_commands.end();
}

void CommandManager::register_command(const String& command_name,
                                      Command command,
                                      const CommandCompleter& completer)
{
    m_commands[command_name] = CommandDescriptor { command, completer };
}

void CommandManager::register_commands(const memoryview<String>& command_names,
                                       Command command,
                                       const CommandCompleter& completer)
{
    for (auto command_name : command_names)
        register_command(command_name, command, completer);
}

namespace
{

struct Token
{
    enum class Type
    {
        Raw,
        ShellExpand,
        RegisterExpand,
        OptionExpand,
        CommandSeparator
    };
    Token() : m_type(Type::Raw) {}

    explicit Token(const String& string) : m_content(string), m_type(Type::Raw) {}
    explicit Token(Type type) : m_type(type) {}
    Token(Type type, String str) : m_content(str), m_type(type) {}

    Type type() const { return m_type; }

    const String& content() const { return m_content; }

private:
    Type   m_type;
    String m_content;
};


using TokenList = std::vector<Token>;
using TokenPosList = std::vector<std::pair<size_t, size_t>>;

bool is_command_separator(Character c)
{
    return c == ';' or c == '\n';
}

bool is_horizontal_blank(char c)
{
   return c == ' ' or c == '\t';
}

TokenList parse(const String& line,
                TokenPosList* opt_token_pos_info = NULL)
{
    TokenList result;

    size_t length = line.length();
    size_t pos = 0;
    while (pos < length)
    {
        while (pos != length)
        {
            if (is_horizontal_blank(line[pos]))
                ++pos;
            else if (line[pos] == '\\' and pos+1 < length and line[pos+1] == '\n')
                pos += 2;
            else
                break;
        }

        size_t token_start = pos;

        Token::Type type = Token::Type::Raw;
        if (line[pos] == '"' or line[pos] == '\'')
        {
            char delimiter = line[pos];

            token_start = ++pos;

            while ((line[pos] != delimiter or line[pos-1] == '\\') and
                    pos != length)
                ++pos;
        }
        else if (line[pos] == '%')
        {
            size_t type_start = ++pos;
            while (isalpha(line[pos]))
                ++pos;
            String type_name = line.substr(type_start, pos - type_start);

            if (type_name == "sh")
                type = Token::Type::ShellExpand;
            if (type_name == "reg")
                type = Token::Type::RegisterExpand;
            if (type_name == "opt")
                type = Token::Type::OptionExpand;

            static const std::unordered_map<Character, Character> matching_delimiters = {
                { '(', ')' }, { '[', ']' }, { '{', '}' }, { '<', '>' }
            };

            Character opening_delimiter = line[pos];
            token_start = ++pos;

            auto delim_it = matching_delimiters.find(opening_delimiter);
            if (delim_it != matching_delimiters.end())
            {
                Character closing_delimiter = delim_it->second;
                int level = 0;
                while (pos != length)
                {
                    if (line[pos-1] != '\\' and line[pos] == opening_delimiter)
                        ++level;
                    if (line[pos-1] != '\\' and line[pos] == closing_delimiter)
                    {
                        if (level > 0)
                            --level;
                        else
                            break;
                    }
                    ++pos;
                }
            }
            else
            {
                while ((line[pos] != opening_delimiter or line[pos-1] == '\\') and
                        pos != length)
                    ++pos;
            }
        }
        else
            while (pos != length and not is_horizontal_blank(line[pos]) and
                   (not is_command_separator(line[pos]) or line[pos-1] == '\\'))
                ++pos;

        if (token_start != pos)
        {
            if (opt_token_pos_info)
                opt_token_pos_info->push_back({token_start, pos});
            result.push_back({type, line.substr(token_start, pos - token_start)});
        }

        if (is_command_separator(line[pos]))
        {
            if (opt_token_pos_info)
                opt_token_pos_info->push_back({pos, pos+1});
            result.push_back(Token{ Token::Type::CommandSeparator });
        }

        ++pos;
    }
    if (not result.empty() and result.back().type() == Token::Type::CommandSeparator)
        result.pop_back();

    return result;<commit_msg>CommandManager::parse: fix invalid memory read<commit_after>#include "command_manager.hh"

#include "utils.hh"
#include "assert.hh"
#include "context.hh"
#include "shell_manager.hh"
#include "register_manager.hh"

#include <algorithm>

namespace Kakoune
{

bool CommandManager::command_defined(const String& command_name) const
{
    return m_commands.find(command_name) != m_commands.end();
}

void CommandManager::register_command(const String& command_name,
                                      Command command,
                                      const CommandCompleter& completer)
{
    m_commands[command_name] = CommandDescriptor { command, completer };
}

void CommandManager::register_commands(const memoryview<String>& command_names,
                                       Command command,
                                       const CommandCompleter& completer)
{
    for (auto command_name : command_names)
        register_command(command_name, command, completer);
}

namespace
{

struct Token
{
    enum class Type
    {
        Raw,
        ShellExpand,
        RegisterExpand,
        OptionExpand,
        CommandSeparator
    };
    Token() : m_type(Type::Raw) {}

    explicit Token(const String& string) : m_content(string), m_type(Type::Raw) {}
    explicit Token(Type type) : m_type(type) {}
    Token(Type type, String str) : m_content(str), m_type(type) {}

    Type type() const { return m_type; }

    const String& content() const { return m_content; }

private:
    Type   m_type;
    String m_content;
};


using TokenList = std::vector<Token>;
using TokenPosList = std::vector<std::pair<size_t, size_t>>;

bool is_command_separator(Character c)
{
    return c == ';' or c == '\n';
}

bool is_horizontal_blank(char c)
{
   return c == ' ' or c == '\t';
}

TokenList parse(const String& line,
                TokenPosList* opt_token_pos_info = NULL)
{
    TokenList result;

    size_t length = line.length();
    size_t pos = 0;
    while (pos < length)
    {
        while (pos != length)
        {
            if (is_horizontal_blank(line[pos]))
                ++pos;
            else if (line[pos] == '\\' and pos+1 < length and line[pos+1] == '\n')
                pos += 2;
            else
                break;
        }

        size_t token_start = pos;

        Token::Type type = Token::Type::Raw;
        if (line[pos] == '"' or line[pos] == '\'')
        {
            char delimiter = line[pos];

            token_start = ++pos;

            while ((line[pos] != delimiter or line[pos-1] == '\\') and
                    pos != length)
                ++pos;
        }
        else if (line[pos] == '%')
        {
            size_t type_start = ++pos;
            while (isalpha(line[pos]))
                ++pos;
            String type_name = line.substr(type_start, pos - type_start);

            if (type_name == "sh")
                type = Token::Type::ShellExpand;
            if (type_name == "reg")
                type = Token::Type::RegisterExpand;
            if (type_name == "opt")
                type = Token::Type::OptionExpand;

            static const std::unordered_map<Character, Character> matching_delimiters = {
                { '(', ')' }, { '[', ']' }, { '{', '}' }, { '<', '>' }
            };

            Character opening_delimiter = line[pos];
            token_start = ++pos;

            auto delim_it = matching_delimiters.find(opening_delimiter);
            if (delim_it != matching_delimiters.end())
            {
                Character closing_delimiter = delim_it->second;
                int level = 0;
                while (pos != length)
                {
                    if (line[pos-1] != '\\' and line[pos] == opening_delimiter)
                        ++level;
                    if (line[pos-1] != '\\' and line[pos] == closing_delimiter)
                    {
                        if (level > 0)
                            --level;
                        else
                            break;
                    }
                    ++pos;
                }
            }
            else
            {
                while (pos != length and
                       (line[pos] != opening_delimiter or line[pos-1] == '\\'))
                    ++pos;
            }
        }
        else
            while (pos != length and not is_horizontal_blank(line[pos]) and
                   (not is_command_separator(line[pos]) or
                    (pos != 0 and line[pos-1] == '\\')))
                ++pos;

        if (token_start != pos)
        {
            if (opt_token_pos_info)
                opt_token_pos_info->push_back({token_start, pos});
            result.push_back({type, line.substr(token_start, pos - token_start)});
        }

        if (is_command_separator(line[pos]))
        {
            if (opt_token_pos_info)
                opt_token_pos_info->push_back({pos, pos+1});
            result.push_back(Token{ Token::Type::CommandSeparator });
        }

        ++pos;
    }
    if (not result.empty() and result.back().type() == Token::Type::CommandSeparator)
        result.pop_back();

    return result;
{'fraction_non_alphanumeric': 0.1255741229356005, 'fraction_numerical': 0.00244307632170429, 'mean_word_length': 1.629496402877698, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6474688', 'n_tokens_mistral': 3001, 'n_tokens_neox': 2867, 'n_words': 618}