<commit_before>#include "command_manager.hh"

#include "alias_registry.hh"
#include "assert.hh"
#include "context.hh"
#include "register_manager.hh"
#include "shell_manager.hh"
#include "utils.hh"
#include "optional.hh"

#include <algorithm>

namespace Kakoune
{

bool CommandManager::command_defined(const String& command_name) const
{
    return m_commands.find(command_name) != m_commands.end();
}

void CommandManager::register_command(String command_name,
                                      Command command,
                                      String docstring,
                                      ParameterDesc param_desc,
                                      CommandFlags flags,
                                      CommandHelper helper,
                                      CommandCompleter completer)
{
    m_commands[command_name] = { std::move(command),
                                 std::move(docstring),
                                 std::move(param_desc),
                                 flags,
                                 std::move(helper),
                                 std::move(completer) };
}

struct parse_error : runtime_error
{
    parse_error(StringView error)
        : runtime_error{format("parse error: {}", error)} {}
};

namespace
{

struct Token
{
    enum class Type
    {
        Raw,
        RawEval,
        ShellExpand,
        RegisterExpand,
        OptionExpand,
        ValExpand,
        CommandSeparator
    };
    Token() : m_type(Type::Raw) {}

    Token(Type type, ByteCount b, ByteCount e, CharCoord coord, String str = "")
        : m_type(type), m_begin(b), m_end(e), m_coord(coord), m_content(str) {}

    Type type() const { return m_type; }
    ByteCount begin() const { return m_begin; }
    ByteCount end() const { return m_end; }
    CharCoord coord() const { return m_coord; }
    const String& content() const { return m_content; }

private:
    Type   m_type;
    ByteCount m_begin;
    ByteCount m_end;
    CharCoord m_coord;
    String m_content;
};

using TokenList = Vector<Token>;

struct Reader
{
public:
    [[gnu::always_inline]]
    char operator*() const { return str[pos]; }

    Reader& operator++()
    {
        if (str[pos++] == '\n')
        {
            ++coord.line;
            coord.column = 0;
        }
        else
            ++coord.column;
        return *this;
    }

    [[gnu::always_inline]]
    explicit operator bool() const { return pos < str.length(); }

    [[gnu::always_inline]]
    StringView substr_from(ByteCount start) const
    {
        return str.substr(start, pos - start);
    }

    Optional<char> peek_next() const
    {
        if (pos+1 != str.length())
            return str[pos+1];
        return {};
    }

    StringView str;
    ByteCount pos;
    CharCoord coord;
};

bool is_command_separator(char c)
{
    return c == ';' or c == '\n';
}

template<typename Func>
String get_until_delimiter(Reader& reader, Func is_delimiter)
{
    auto beg = reader.pos;
    String str;
    bool was_antislash = false;

    while (reader)
    {
        const char c = *reader;
        if (is_delimiter(c))
        {
            str += reader.substr_from(beg);
            if (was_antislash)
            {
                str.back() = c;
                beg = reader.pos+1;
            }
            else
                return str;
        }
        was_antislash = c == '\\';
        ++reader;
    }
    if (beg < reader.str.length())
        str += reader.substr_from(beg);
    return str;
}

[[gnu::always_inline]]
inline String get_until_delimiter(Reader& reader, char c)
{
    return get_until_delimiter(reader, [c](char ch) { return c == ch; });
}

StringView get_until_closing_delimiter(Reader& reader, char opening_delimiter,
                                       char closing_delimiter)
{
    kak_assert(reader.str[reader.pos-1] == opening_delimiter);
    int level = 0;
    auto start = reader.pos;
    while (reader)
    {
        const char c = *reader;
        if (c == opening_delimiter)
            ++level;
        else if (c == closing_delimiter)
        {
            if (level > 0)
                --level;
            else
                break;
        }
        ++reader;
    }
    return reader.substr_from(start);
}

struct unknown_expand : parse_error
{
    unknown_expand(StringView name)
        : parse_error{format("unknown expand '{}'", name)} {}
};

template<bool throw_on_invalid>
Token::Type token_type(StringView type_name)
{
    if (type_name == "")
        return Token::Type::Raw;
    else if (type_name == "sh")
        return Token::Type::ShellExpand;
    else if (type_name == "reg")
        return Token::Type::RegisterExpand;
    else if (type_name == "opt")
        return Token::Type::OptionExpand;
    else if (type_name == "val")
        return Token::Type::ValExpand;
    else if (throw_on_invalid)
        throw unknown_expand{type_name};
    else
        return Token::Type::Raw;
}

void skip_blanks_and_comments(Reader& reader)
{
    while (reader)
    {
        const char c = *reader;
        if (is_horizontal_blank(c))
            ++reader;
        else if (c == '\\' and reader.peek_next().value_or('\0') == '\n')
            ++(++reader);
        else if (c == '#')
        {
            for (bool eol = false; reader and not eol; ++reader)
                eol = *reader == '\n';
        }
        else
            break;
    }
}

template<bool throw_on_unterminated>
Token parse_percent_token(Reader& reader)
{
    ++reader;
    const ByteCount type_start = reader.pos;
    while (reader and isalpha(*reader))
        ++reader;
    StringView type_name = reader.substr_from(type_start);

    if (throw_on_unterminated and not reader)
        throw parse_error{format("expected a string delimiter after '%{}'",
                                 type_name)};

    Token::Type type = token_type<throw_on_unterminated>(type_name);
    static const UnorderedMap<char, char> matching_delimiters = {
        { '(', ')' }, { '[', ']' }, { '{', '}' }, { '<', '>' }
    };

    char opening_delimiter = *reader;
    ++reader;
    auto start = reader.pos;
    auto coord = reader.coord;

    auto delim_it = matching_delimiters.find(opening_delimiter);
    if (delim_it != matching_delimiters.end())
    {
        const char closing_delimiter = delim_it->second;
        auto token = get_until_closing_delimiter(reader, opening_delimiter,
                                                 closing_delimiter);
        if (throw_on_unterminated and not reader)
            throw parse_error{format("{}:{}: unterminated string '%{}{}...{}'",
                                     coord.line, coord.column, type_name,
                                     opening_delimiter, closing_delimiter)};

        return {type, start, reader.pos, coord, token.str()};
    }
    else
    {
        String token = get_until_delimiter(reader, opening_delimiter);

        if (throw_on_unterminated and not reader)
            throw parse_error{format("{}:{}: unterminated string '%{}{}...{}'",
                                     coord.line, coord.column, type_name,
                                     opening_delimiter, opening_delimiter)};

        return {type, start, reader.pos, coord, std::move(token)};
    }
}
<commit_msg>Avoid using an UnorderedMap for 4 elements...<commit_after>#include "command_manager.hh"

#include "alias_registry.hh"
#include "assert.hh"
#include "context.hh"
#include "register_manager.hh"
#include "shell_manager.hh"
#include "utils.hh"
#include "optional.hh"

#include <algorithm>

namespace Kakoune
{

bool CommandManager::command_defined(const String& command_name) const
{
    return m_commands.find(command_name) != m_commands.end();
}

void CommandManager::register_command(String command_name,
                                      Command command,
                                      String docstring,
                                      ParameterDesc param_desc,
                                      CommandFlags flags,
                                      CommandHelper helper,
                                      CommandCompleter completer)
{
    m_commands[command_name] = { std::move(command),
                                 std::move(docstring),
                                 std::move(param_desc),
                                 flags,
                                 std::move(helper),
                                 std::move(completer) };
}

struct parse_error : runtime_error
{
    parse_error(StringView error)
        : runtime_error{format("parse error: {}", error)} {}
};

namespace
{

struct Token
{
    enum class Type
    {
        Raw,
        RawEval,
        ShellExpand,
        RegisterExpand,
        OptionExpand,
        ValExpand,
        CommandSeparator
    };
    Token() : m_type(Type::Raw) {}

    Token(Type type, ByteCount b, ByteCount e, CharCoord coord, String str = "")
        : m_type(type), m_begin(b), m_end(e), m_coord(coord), m_content(str) {}

    Type type() const { return m_type; }
    ByteCount begin() const { return m_begin; }
    ByteCount end() const { return m_end; }
    CharCoord coord() const { return m_coord; }
    const String& content() const { return m_content; }

private:
    Type   m_type;
    ByteCount m_begin;
    ByteCount m_end;
    CharCoord m_coord;
    String m_content;
};

using TokenList = Vector<Token>;

struct Reader
{
public:
    [[gnu::always_inline]]
    char operator*() const { return str[pos]; }

    Reader& operator++()
    {
        if (str[pos++] == '\n')
        {
            ++coord.line;
            coord.column = 0;
        }
        else
            ++coord.column;
        return *this;
    }

    [[gnu::always_inline]]
    explicit operator bool() const { return pos < str.length(); }

    [[gnu::always_inline]]
    StringView substr_from(ByteCount start) const
    {
        return str.substr(start, pos - start);
    }

    Optional<char> peek_next() const
    {
        if (pos+1 != str.length())
            return str[pos+1];
        return {};
    }

    StringView str;
    ByteCount pos;
    CharCoord coord;
};

bool is_command_separator(char c)
{
    return c == ';' or c == '\n';
}

template<typename Func>
String get_until_delimiter(Reader& reader, Func is_delimiter)
{
    auto beg = reader.pos;
    String str;
    bool was_antislash = false;

    while (reader)
    {
        const char c = *reader;
        if (is_delimiter(c))
        {
            str += reader.substr_from(beg);
            if (was_antislash)
            {
                str.back() = c;
                beg = reader.pos+1;
            }
            else
                return str;
        }
        was_antislash = c == '\\';
        ++reader;
    }
    if (beg < reader.str.length())
        str += reader.substr_from(beg);
    return str;
}

[[gnu::always_inline]]
inline String get_until_delimiter(Reader& reader, char c)
{
    return get_until_delimiter(reader, [c](char ch) { return c == ch; });
}

StringView get_until_closing_delimiter(Reader& reader, char opening_delimiter,
                                       char closing_delimiter)
{
    kak_assert(reader.str[reader.pos-1] == opening_delimiter);
    int level = 0;
    auto start = reader.pos;
    while (reader)
    {
        const char c = *reader;
        if (c == opening_delimiter)
            ++level;
        else if (c == closing_delimiter)
        {
            if (level > 0)
                --level;
            else
                break;
        }
        ++reader;
    }
    return reader.substr_from(start);
}

struct unknown_expand : parse_error
{
    unknown_expand(StringView name)
        : parse_error{format("unknown expand '{}'", name)} {}
};

template<bool throw_on_invalid>
Token::Type token_type(StringView type_name)
{
    if (type_name == "")
        return Token::Type::Raw;
    else if (type_name == "sh")
        return Token::Type::ShellExpand;
    else if (type_name == "reg")
        return Token::Type::RegisterExpand;
    else if (type_name == "opt")
        return Token::Type::OptionExpand;
    else if (type_name == "val")
        return Token::Type::ValExpand;
    else if (throw_on_invalid)
        throw unknown_expand{type_name};
    else
        return Token::Type::Raw;
}

void skip_blanks_and_comments(Reader& reader)
{
    while (reader)
    {
        const char c = *reader;
        if (is_horizontal_blank(c))
            ++reader;
        else if (c == '\\' and reader.peek_next().value_or('\0') == '\n')
            ++(++reader);
        else if (c == '#')
        {
            for (bool eol = false; reader and not eol; ++reader)
                eol = *reader == '\n';
        }
        else
            break;
    }
}

template<bool throw_on_unterminated>
Token parse_percent_token(Reader& reader)
{
    ++reader;
    const ByteCount type_start = reader.pos;
    while (reader and isalpha(*reader))
        ++reader;
    StringView type_name = reader.substr_from(type_start);

    if (throw_on_unterminated and not reader)
        throw parse_error{format("expected a string delimiter after '%{}'",
                                 type_name)};

    Token::Type type = token_type<throw_on_unterminated>(type_name);

    constexpr struct CharPair { char opening; char closing; } matching_pairs[] = {
        { '(', ')' }, { '[', ']' }, { '{', '}' }, { '<', '>' }
    };

    char opening_delimiter = *reader;
    ++reader;
    auto start = reader.pos;
    auto coord = reader.coord;

    auto it = find_if(matching_pairs, [opening_delimiter](const CharPair& cp)
                      { return opening_delimiter == cp.opening; });

    if (it != std::end(matching_pairs))
    {
        const char closing_delimiter = it->closing;
        auto token = get_until_closing_delimiter(reader, opening_delimiter,
                                                 closing_delimiter);
        if (throw_on_unterminated and not reader)
            throw parse_error{format("{}:{}: unterminated string '%{}{}...{}'",
                                     coord.line, coord.column, type_name,
                                     opening_delimiter, closing_delimiter)};

        return {type, start, reader.pos, coord, token.str()};
    }
    else
    {
        String token = get_until_delimiter(reader, opening_delimiter);

        if (throw_on_unterminated and not reader)
            throw parse_error{format("{}:{}: unterminated string '%{}{}...{}'",
                                     coord.line, coord.column, type_name,
                                     opening_delimiter, opening_delimiter)};

        return {type, start, reader.pos, coord, std::move(token)};
    }
}

{'fraction_non_alphanumeric': 0.11970584229878796, 'fraction_numerical': 0.0011575650279177449, 'mean_word_length': 1.9274466812836357, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1965838', 'n_tokens_mistral': 4350, 'n_tokens_neox': 4244, 'n_words': 971}