<commit_before>#include "insert_completer.hh"

#include "buffer_manager.hh"
#include "buffer_utils.hh"
#include "client.hh"
#include "changes.hh"
#include "context.hh"
#include "display_buffer.hh"
#include "face_registry.hh"
#include "file.hh"
#include "regex.hh"
#include "window.hh"
#include "word_db.hh"
#include "utf8_iterator.hh"
#include "user_interface.hh"

#include <numeric>
#include <utility>

namespace Kakoune
{

using StringList = Vector<String, MemoryDomain::Options>;

String option_to_string(const InsertCompleterDesc& opt)
{
    switch (opt.mode)
    {
        case InsertCompleterDesc::Word:
            return "word=" + (opt.param ? *opt.param : "");
        case InsertCompleterDesc::Filename:
            return "filename";
        case InsertCompleterDesc::Option:
            return "option=" + (opt.param ? *opt.param : "");
        case InsertCompleterDesc::Line:
            return "line";
    }
    kak_assert(false);
    return "";
}

void option_from_string(StringView str, InsertCompleterDesc& opt)
{
    if (str.substr(0_byte, 7_byte) == "option=")
    {
        opt.mode = InsertCompleterDesc::Option;
        opt.param = str.substr(7_byte).str();
        return;
    }
    else if (str.substr(0_byte, 5_byte) == "word=")
    {
        auto param = str.substr(5_byte);
        if (param == "all" or param == "buffer")
        {
            opt.mode = InsertCompleterDesc::Word;
            opt.param = param.str();
            return;
        }
    }
    else if (str == "filename")
    {
        opt.mode = InsertCompleterDesc::Filename;
        opt.param = Optional<String>{};
        return;
    }
    else if (str == "line")
    {
        opt.mode = InsertCompleterDesc::Line;
        opt.param = Optional<String>{};
        return;
    }
    throw runtime_error(format("invalid completer description: '{}'", str));
}

namespace
{

WordDB& get_word_db(const Buffer& buffer)
{
    static const ValueId word_db_id = get_free_value_id();
    Value& cache_val = buffer.values()[word_db_id];
    if (not cache_val)
        cache_val = Value(WordDB{buffer});
    return cache_val.as<WordDB>();
}

template<bool other_buffers>
InsertCompletion complete_word(const SelectionList& sels, const OptionManager& options)
{
    ConstArrayView<Codepoint> extra_word_chars = options["extra_word_chars"].get<Vector<Codepoint, MemoryDomain::Options>>();
    auto is_word_pred = [extra_word_chars](Codepoint c) { return is_word(c, extra_word_chars); };

    const Buffer& buffer = sels.buffer();
    BufferCoord cursor_pos = sels.main().cursor();

    using Utf8It = utf8::iterator<BufferIterator>;
    Utf8It pos{buffer.iterator_at(cursor_pos), buffer};
    if (pos == buffer.begin() or not is_word_pred(*(pos-1)))
        return {};

    BufferCoord word_begin;
    StringView prefix;
    HashMap<StringView, int> sel_word_counts;
    for (int i = 0; i < sels.size(); ++i)
    {
        Utf8It end{buffer.iterator_at(sels[i].cursor()), buffer};
        Utf8It begin = end-1;
        if (not skip_while_reverse(begin, buffer.begin(), is_word_pred) and
            begin < end) // (begin might == end if end == buffer.begin())
            ++begin;

        if (i == sels.main_index())
        {
            word_begin = begin.base().coord();
            prefix = buffer.substr(word_begin, end.base().coord());
        }

        skip_while(end, buffer.end(), is_word_pred);

        StringView word = buffer.substr(begin.base().coord(), end.base().coord());
        ++sel_word_counts[word];
    }

    struct RankedMatchAndBuffer : RankedMatch
    {
        RankedMatchAndBuffer(RankedMatch  m, const Buffer* b)
            : RankedMatch{std::move(m)}, buffer{b} {}

        using RankedMatch::operator==;
        using RankedMatch::operator<;
        bool operator==(StringView other) const { return candidate() == other; }

        const Buffer* buffer;
    };
    Vector<RankedMatchAndBuffer> matches;

    auto& word_db = get_word_db(buffer);
    for (auto& m : word_db.find_matching(prefix))
        matches.push_back({ m, &buffer });

    // Remove words that are being edited
    for (auto& word_count : sel_word_counts)
    {
        if (word_db.get_word_occurences(word_count.key) <= word_count.value)
            unordered_erase(matches, word_count.key);
    }

    if (other_buffers)
    {
        for (const auto& buf : BufferManager::instance())
        {
            if (buf.get() == &buffer or buf->flags() & Buffer::Flags::Debug)
                continue;
            for (auto& m : get_word_db(*buf).find_matching(prefix) |
                           // filter out words that are not considered words for the current buffer
                           filter([&](auto& rm) { return std::all_of(rm.candidate().begin(), rm.candidate().end(),
                                                                     is_word_pred); }))
                matches.push_back({ m, buf.get() });
        }
    }

    using StaticWords = Vector<String, MemoryDomain::Options>;
    for (auto& word : options["static_words"].get<StaticWords>())
        if (RankedMatch match{word, prefix})
            matches.emplace_back(match, nullptr);

    unordered_erase(matches, prefix);
    const auto longest = accumulate(matches, 0_char,
                                    [](const CharCount& lhs, const RankedMatchAndBuffer& rhs)
                                    { return std::max(lhs, rhs.candidate().char_length()); });

    constexpr size_t max_count = 100;
    // Gather best max_count matches
    auto greater = [](auto& lhs, auto& rhs) { return rhs < lhs; };
    auto first = matches.begin(), last = matches.end();
    std::make_heap(first, last, greater);
    InsertCompletion::CandidateList candidates;
    candidates.reserve(std::min(matches.size(), max_count));
    while (candidates.size() < max_count and first != last)
    {
        if (candidates.empty() or candidates.back().completion != first->candidate())
        {
            DisplayLine menu_entry;
            if (first->buffer)<commit_msg>Insert complete: Remove path info with one buffer<commit_after>#include "insert_completer.hh"

#include "buffer_manager.hh"
#include "buffer_utils.hh"
#include "client.hh"
#include "changes.hh"
#include "context.hh"
#include "display_buffer.hh"
#include "face_registry.hh"
#include "file.hh"
#include "regex.hh"
#include "window.hh"
#include "word_db.hh"
#include "utf8_iterator.hh"
#include "user_interface.hh"

#include <numeric>
#include <utility>

namespace Kakoune
{

using StringList = Vector<String, MemoryDomain::Options>;

String option_to_string(const InsertCompleterDesc& opt)
{
    switch (opt.mode)
    {
        case InsertCompleterDesc::Word:
            return "word=" + (opt.param ? *opt.param : "");
        case InsertCompleterDesc::Filename:
            return "filename";
        case InsertCompleterDesc::Option:
            return "option=" + (opt.param ? *opt.param : "");
        case InsertCompleterDesc::Line:
            return "line";
    }
    kak_assert(false);
    return "";
}

void option_from_string(StringView str, InsertCompleterDesc& opt)
{
    if (str.substr(0_byte, 7_byte) == "option=")
    {
        opt.mode = InsertCompleterDesc::Option;
        opt.param = str.substr(7_byte).str();
        return;
    }
    else if (str.substr(0_byte, 5_byte) == "word=")
    {
        auto param = str.substr(5_byte);
        if (param == "all" or param == "buffer")
        {
            opt.mode = InsertCompleterDesc::Word;
            opt.param = param.str();
            return;
        }
    }
    else if (str == "filename")
    {
        opt.mode = InsertCompleterDesc::Filename;
        opt.param = Optional<String>{};
        return;
    }
    else if (str == "line")
    {
        opt.mode = InsertCompleterDesc::Line;
        opt.param = Optional<String>{};
        return;
    }
    throw runtime_error(format("invalid completer description: '{}'", str));
}

namespace
{

WordDB& get_word_db(const Buffer& buffer)
{
    static const ValueId word_db_id = get_free_value_id();
    Value& cache_val = buffer.values()[word_db_id];
    if (not cache_val)
        cache_val = Value(WordDB{buffer});
    return cache_val.as<WordDB>();
}

template<bool other_buffers>
InsertCompletion complete_word(const SelectionList& sels, const OptionManager& options)
{
    ConstArrayView<Codepoint> extra_word_chars = options["extra_word_chars"].get<Vector<Codepoint, MemoryDomain::Options>>();
    auto is_word_pred = [extra_word_chars](Codepoint c) { return is_word(c, extra_word_chars); };

    const Buffer& buffer = sels.buffer();
    BufferCoord cursor_pos = sels.main().cursor();

    using Utf8It = utf8::iterator<BufferIterator>;
    Utf8It pos{buffer.iterator_at(cursor_pos), buffer};
    if (pos == buffer.begin() or not is_word_pred(*(pos-1)))
        return {};

    BufferCoord word_begin;
    StringView prefix;
    HashMap<StringView, int> sel_word_counts;
    for (int i = 0; i < sels.size(); ++i)
    {
        Utf8It end{buffer.iterator_at(sels[i].cursor()), buffer};
        Utf8It begin = end-1;
        if (not skip_while_reverse(begin, buffer.begin(), is_word_pred) and
            begin < end) // (begin might == end if end == buffer.begin())
            ++begin;

        if (i == sels.main_index())
        {
            word_begin = begin.base().coord();
            prefix = buffer.substr(word_begin, end.base().coord());
        }

        skip_while(end, buffer.end(), is_word_pred);

        StringView word = buffer.substr(begin.base().coord(), end.base().coord());
        ++sel_word_counts[word];
    }

    struct RankedMatchAndBuffer : RankedMatch
    {
        RankedMatchAndBuffer(RankedMatch  m, const Buffer* b)
            : RankedMatch{std::move(m)}, buffer{b} {}

        using RankedMatch::operator==;
        using RankedMatch::operator<;
        bool operator==(StringView other) const { return candidate() == other; }

        const Buffer* buffer;
    };
    Vector<RankedMatchAndBuffer> matches;

    auto& word_db = get_word_db(buffer);
    for (auto& m : word_db.find_matching(prefix))
        matches.push_back({ m, &buffer });

    // Remove words that are being edited
    for (auto& word_count : sel_word_counts)
    {
        if (word_db.get_word_occurences(word_count.key) <= word_count.value)
            unordered_erase(matches, word_count.key);
    }

    if (other_buffers)
    {
        for (const auto& buf : BufferManager::instance())
        {
            if (buf.get() == &buffer or buf->flags() & Buffer::Flags::Debug)
                continue;
            for (auto& m : get_word_db(*buf).find_matching(prefix) |
                           // filter out words that are not considered words for the current buffer
                           filter([&](auto& rm) { return std::all_of(rm.candidate().begin(), rm.candidate().end(),
                                                                     is_word_pred); }))
                matches.push_back({ m, buf.get() });
        }
    }

    using StaticWords = Vector<String, MemoryDomain::Options>;
    for (auto& word : options["static_words"].get<StaticWords>())
        if (RankedMatch match{word, prefix})
            matches.emplace_back(match, nullptr);

    unordered_erase(matches, prefix);
    const auto longest = accumulate(matches, 0_char,
                                    [](const CharCount& lhs, const RankedMatchAndBuffer& rhs)
                                    { return std::max(lhs, rhs.candidate().char_length()); });

    constexpr size_t max_count = 100;
    // Gather best max_count matches
    auto greater = [](auto& lhs, auto& rhs) { return rhs < lhs; };
    auto first = matches.begin(), last = matches.end();
    std::make_heap(first, last, greater);
    InsertCompletion::CandidateList candidates;
    candidates.reserve(std::min(matches.size(), max_count));
    while (candidates.size() < max_count and first != last)
    {
        if (candidates.empty() or candidates.back().completion != first->candidate())
        {
            DisplayLine menu_entry;
            if (other_buffers && first->buffer)
{'fraction_non_alphanumeric': 0.1339116067740603, 'fraction_numerical': 0.0031391986782321355, 'mean_word_length': 2.648583484026522, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2608199', 'n_tokens_mistral': 3728, 'n_tokens_neox': 3680, 'n_words': 801}