<commit_before>#include "editor.hh"

#include "exception.hh"
#include "utils.hh"
#include "register.hh"
#include "register_manager.hh"

#include "utf8_iterator.hh"

#include <array>

namespace Kakoune
{

Editor::Editor(Buffer& buffer)
    : m_buffer(&buffer),
      m_edition_level(0),
      m_selections(buffer)
{
    m_selections.push_back(Selection(buffer.begin(), buffer.begin()));
    m_main_sel = 0;
}

void Editor::erase()
{
    scoped_edition edition(*this);
    for (auto& sel : m_selections)
    {
        m_buffer->erase(sel.begin(), sel.end());
        sel.avoid_eol();
    }
}

static BufferIterator prepare_insert(Buffer& buffer, const Selection& sel,
                                     InsertMode mode)
{
    switch (mode)
    {
    case InsertMode::Insert:
        return sel.begin();
    case InsertMode::Replace:
    {
        BufferIterator pos = sel.begin();
        buffer.erase(sel.begin(), sel.end());
        return pos;
    }
    case InsertMode::Append:
    {
        // special case for end of lines, append to current line instead
        auto pos = std::max(sel.first(), sel.last());
        if (pos.column() == buffer.line_length(pos.line()) - 1)
            return pos;
        else
            return pos+1;<commit_msg>Editor: fix utf8 handling in prepare_insert<commit_after>#include "editor.hh"

#include "exception.hh"
#include "utils.hh"
#include "register.hh"
#include "register_manager.hh"

#include "utf8_iterator.hh"

#include <array>

namespace Kakoune
{

Editor::Editor(Buffer& buffer)
    : m_buffer(&buffer),
      m_edition_level(0),
      m_selections(buffer)
{
    m_selections.push_back(Selection(buffer.begin(), buffer.begin()));
    m_main_sel = 0;
}

void Editor::erase()
{
    scoped_edition edition(*this);
    for (auto& sel : m_selections)
    {
        m_buffer->erase(sel.begin(), sel.end());
        sel.avoid_eol();
    }
}

static BufferIterator prepare_insert(Buffer& buffer, const Selection& sel,
                                     InsertMode mode)
{
    switch (mode)
    {
    case InsertMode::Insert:
        return sel.begin();
    case InsertMode::Replace:
    {
        BufferIterator pos = sel.begin();
        buffer.erase(sel.begin(), sel.end());
        return pos;
    }
    case InsertMode::Append:
    {
        // special case for end of lines, append to current line instead
        auto pos = std::max(sel.first(), sel.last());
        if (pos.column() == buffer.line_length(pos.line()) - 1)
            return pos;
        else
            return utf8::next(pos);
{'fraction_non_alphanumeric': 0.1358994501178319, 'fraction_numerical': 0.00432050274941084, 'mean_word_length': 2.597457627118644, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '565475', 'n_tokens_mistral': 805, 'n_tokens_neox': 801, 'n_words': 176}