<commit_before>#include "command_manager.hh"

#include "alias_registry.hh"
#include "assert.hh"
#include "context.hh"
#include "register_manager.hh"
#include "shell_manager.hh"
#include "utils.hh"

#include <algorithm>

namespace Kakoune
{

bool CommandManager::command_defined(const String& command_name) const
{
    return m_commands.find(command_name) != m_commands.end();
}

void CommandManager::register_command(String command_name,
                                      Command command,
                                      String docstring,
                                      ParameterDesc param_desc,
                                      CommandFlags flags,
                                      CommandHelper helper,
                                      CommandCompleter completer)
{
    m_commands[command_name] = { std::move(command),
                                 std::move(docstring),
                                 std::move(param_desc),
                                 flags,
                                 std::move(helper),
                                 std::move(completer) };
}

struct parse_error : runtime_error
{
    parse_error(StringView error)
        : runtime_error{format("parse error: {}", error)} {}
};

namespace
{

struct Token
{
    enum class Type
    {
        Raw,
        RawEval,
        ShellExpand,
        RegisterExpand,
        OptionExpand,
        ValExpand,
        CommandSeparator
    };
    Token() : m_type(Type::Raw) {}

    Token(Type type, ByteCount b, ByteCount e, String str = "")
    : m_type(type), m_begin(b), m_end(e), m_content(str) {}

    Type type() const { return m_type; }
    ByteCount begin() const { return m_begin; }
    ByteCount end() const { return m_end; }
    const String& content() const { return m_content; }

private:
    Type   m_type;
    ByteCount m_begin;
    ByteCount m_end;
    String m_content;
};


using TokenList = Vector<Token>;

bool is_command_separator(char c)
{
    return c == ';' or c == '\n';
}

String get_until_delimiter(StringView base, ByteCount& pos, char delimiter)
{
    const ByteCount length = base.length();
    ByteCount beg = pos;
    String str;
    while (pos < length)
    {
        char c = base[pos];
        if (c == delimiter)
        {
            str += base.substr(beg, pos - beg);
            if (pos != 0 and base[pos-1] == '\\')
            {
                str.back() = delimiter;
                beg = pos+1;
            }
            else
                return str;
        }
        ++pos;
    }
    str += base.substr(beg, pos - beg);
    return str;
}

String get_until_delimiter(StringView base, ByteCount& pos,
                           char opening_delimiter, char closing_delimiter)
{
    kak_assert(base[pos-1] == opening_delimiter);
    const ByteCount length = base.length();
    int level = 0;
    ByteCount start = pos;
    while (pos != length)
    {
        if (base[pos] == opening_delimiter)
            ++level;
        else if (base[pos] == closing_delimiter)
        {
            if (level > 0)
                --level;
            else
                break;
        }
        ++pos;
    }
    return base.substr(start, pos - start).str();
}

struct unknown_expand : parse_error
{
    unknown_expand(StringView name)
        : parse_error{format("unknown expand '{}'", name)} {}
};

template<bool throw_on_invalid>
Token::Type token_type(StringView type_name)
{
    if (type_name == "")
        return Token::Type::Raw;
    else if (type_name == "sh")
        return Token::Type::ShellExpand;
    else if (type_name == "reg")
        return Token::Type::RegisterExpand;
    else if (type_name == "opt")
        return Token::Type::OptionExpand;
    else if (type_name == "val")
        return Token::Type::ValExpand;
    else if (throw_on_invalid)
        throw unknown_expand{type_name};
    else
        return Token::Type::Raw;
}

void skip_blanks_and_comments(StringView base, ByteCount& pos)
{
    const ByteCount length = base.length();
    while (pos != length)
    {
        if (is_horizontal_blank(base[pos]))
            ++pos;
        else if (base[pos] == '\\' and pos+1 < length and base[pos+1] == '\n')
            pos += 2;
        else if (base[pos] == '#')
        {
            while (pos != length)
            {
                if (base[pos++] == '\n')
                    break;
            }
        }
        else
            break;
    }
}

template<bool throw_on_unterminated>
Token parse_percent_token(StringView line, ByteCount& pos)
{
    const ByteCount length = line.length();
    const ByteCount type_start = ++pos;
    while (isalpha(line[pos]))
        ++pos;
    StringView type_name = line.substr(type_start, pos - type_start);

    if (throw_on_unterminated and pos == length)
        throw parse_error{format("expected a string delimiter after '%{}'",
                                 type_name)};

    Token::Type type = token_type<throw_on_unterminated>(type_name);
    static const UnorderedMap<char, char> matching_delimiters = {
        { '(', ')' }, { '[', ']' }, { '{', '}' }, { '<', '>' }
    };

    char opening_delimiter = line[pos];
    ByteCount token_start = ++pos;

    auto delim_it = matching_delimiters.find(opening_delimiter);
    if (delim_it != matching_delimiters.end())<commit_msg>Fix command parsing bug<commit_after>#include "command_manager.hh"

#include "alias_registry.hh"
#include "assert.hh"
#include "context.hh"
#include "register_manager.hh"
#include "shell_manager.hh"
#include "utils.hh"

#include <algorithm>

namespace Kakoune
{

bool CommandManager::command_defined(const String& command_name) const
{
    return m_commands.find(command_name) != m_commands.end();
}

void CommandManager::register_command(String command_name,
                                      Command command,
                                      String docstring,
                                      ParameterDesc param_desc,
                                      CommandFlags flags,
                                      CommandHelper helper,
                                      CommandCompleter completer)
{
    m_commands[command_name] = { std::move(command),
                                 std::move(docstring),
                                 std::move(param_desc),
                                 flags,
                                 std::move(helper),
                                 std::move(completer) };
}

struct parse_error : runtime_error
{
    parse_error(StringView error)
        : runtime_error{format("parse error: {}", error)} {}
};

namespace
{

struct Token
{
    enum class Type
    {
        Raw,
        RawEval,
        ShellExpand,
        RegisterExpand,
        OptionExpand,
        ValExpand,
        CommandSeparator
    };
    Token() : m_type(Type::Raw) {}

    Token(Type type, ByteCount b, ByteCount e, String str = "")
    : m_type(type), m_begin(b), m_end(e), m_content(str) {}

    Type type() const { return m_type; }
    ByteCount begin() const { return m_begin; }
    ByteCount end() const { return m_end; }
    const String& content() const { return m_content; }

private:
    Type   m_type;
    ByteCount m_begin;
    ByteCount m_end;
    String m_content;
};


using TokenList = Vector<Token>;

bool is_command_separator(char c)
{
    return c == ';' or c == '\n';
}

String get_until_delimiter(StringView base, ByteCount& pos, char delimiter)
{
    const ByteCount length = base.length();
    ByteCount beg = pos;
    String str;
    while (pos < length)
    {
        char c = base[pos];
        if (c == delimiter)
        {
            str += base.substr(beg, pos - beg);
            if (pos != 0 and base[pos-1] == '\\')
            {
                str.back() = delimiter;
                beg = pos+1;
            }
            else
                return str;
        }
        ++pos;
    }
    if (beg < length)
        str += base.substr(beg, pos - beg);
    return str;
}

String get_until_delimiter(StringView base, ByteCount& pos,
                           char opening_delimiter, char closing_delimiter)
{
    kak_assert(base[pos-1] == opening_delimiter);
    const ByteCount length = base.length();
    int level = 0;
    ByteCount start = pos;
    while (pos != length)
    {
        if (base[pos] == opening_delimiter)
            ++level;
        else if (base[pos] == closing_delimiter)
        {
            if (level > 0)
                --level;
            else
                break;
        }
        ++pos;
    }
    return base.substr(start, pos - start).str();
}

struct unknown_expand : parse_error
{
    unknown_expand(StringView name)
        : parse_error{format("unknown expand '{}'", name)} {}
};

template<bool throw_on_invalid>
Token::Type token_type(StringView type_name)
{
    if (type_name == "")
        return Token::Type::Raw;
    else if (type_name == "sh")
        return Token::Type::ShellExpand;
    else if (type_name == "reg")
        return Token::Type::RegisterExpand;
    else if (type_name == "opt")
        return Token::Type::OptionExpand;
    else if (type_name == "val")
        return Token::Type::ValExpand;
    else if (throw_on_invalid)
        throw unknown_expand{type_name};
    else
        return Token::Type::Raw;
}

void skip_blanks_and_comments(StringView base, ByteCount& pos)
{
    const ByteCount length = base.length();
    while (pos != length)
    {
        if (is_horizontal_blank(base[pos]))
            ++pos;
        else if (base[pos] == '\\' and pos+1 < length and base[pos+1] == '\n')
            pos += 2;
        else if (base[pos] == '#')
        {
            while (pos != length)
            {
                if (base[pos++] == '\n')
                    break;
            }
        }
        else
            break;
    }
}

template<bool throw_on_unterminated>
Token parse_percent_token(StringView line, ByteCount& pos)
{
    const ByteCount length = line.length();
    const ByteCount type_start = ++pos;
    while (pos < length and isalpha(line[pos]))
        ++pos;
    StringView type_name = line.substr(type_start, pos - type_start);

    if (throw_on_unterminated and pos == length)
        throw parse_error{format("expected a string delimiter after '%{}'",
                                 type_name)};

    Token::Type type = token_type<throw_on_unterminated>(type_name);
    static const UnorderedMap<char, char> matching_delimiters = {
        { '(', ')' }, { '[', ']' }, { '{', '}' }, { '<', '>' }
    };

    char opening_delimiter = line[pos];
    ByteCount token_start = ++pos;

    auto delim_it = matching_delimiters.find(opening_delimiter);
    if (delim_it != matching_delimiters.end())
{'fraction_non_alphanumeric': 0.11999625011718384, 'fraction_numerical': 0.0016874472672728977, 'mean_word_length': 1.933993399339934, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '318457', 'n_tokens_mistral': 3176, 'n_tokens_neox': 3101, 'n_words': 722}