<commit_before>#include "highlighters.hh"

#include "assert.hh"
#include "buffer_utils.hh"
#include "changes.hh"
#include "command_manager.hh"
#include "context.hh"
#include "display_buffer.hh"
#include "face_registry.hh"
#include "highlighter_group.hh"
#include "line_modification.hh"
#include "option.hh"
#include "option_types.hh"
#include "parameters_parser.hh"
#include "ranges.hh"
#include "regex.hh"
#include "register_manager.hh"
#include "string.hh"
#include "utf8.hh"
#include "utf8_iterator.hh"
#include "window.hh"

#include <locale>
#include <cstdio>

namespace Kakoune
{

template<typename Func>
std::unique_ptr<Highlighter> make_highlighter(Func func, HighlightPass pass = HighlightPass::Colorize)
{
    struct SimpleHighlighter : public Highlighter
    {
        SimpleHighlighter(Func func, HighlightPass pass)
          : Highlighter{pass}, m_func{std::move(func)} {}

    private:
        void do_highlight(HighlightContext context, DisplayBuffer& display_buffer, BufferRange range) override
        {
            m_func(context, display_buffer, range);
        }
        Func m_func;
    };
    return std::make_unique<SimpleHighlighter>(std::move(func), pass);
}

template<typename T>
void highlight_range(DisplayBuffer& display_buffer,
                     BufferCoord begin, BufferCoord end,
                     bool skip_replaced, T func)
{
    // tolerate begin > end as that can be triggered by wrong encodings
    if (begin >= end or end <= display_buffer.range().begin
                     or begin >= display_buffer.range().end)
        return;

    for (auto& line : display_buffer.lines())
    {
        auto& range = line.range();
        if (range.end <= begin or  end < range.begin)
            continue;

        for (auto atom_it = line.begin(); atom_it != line.end(); ++atom_it)
        {
            bool is_replaced = atom_it->type() == DisplayAtom::ReplacedRange;

            if (not atom_it->has_buffer_range() or
                (skip_replaced and is_replaced) or
                end <= atom_it->begin() or begin >= atom_it->end())
                continue;

            if (not is_replaced and begin > atom_it->begin())
                atom_it = ++line.split(atom_it, begin);

            if (not is_replaced and end < atom_it->end())
            {
                atom_it = line.split(atom_it, end);
                func(*atom_it);
                ++atom_it;
            }
            else
                func(*atom_it);
        }
    }
}

template<typename T>
void replace_range(DisplayBuffer& display_buffer,
                   BufferCoord begin, BufferCoord end, T func)
{
    // tolerate begin > end as that can be triggered by wrong encodings
    if (begin >= end or end <= display_buffer.range().begin
                     or begin >= display_buffer.range().end)
        return;

    for (auto& line : display_buffer.lines())
    {
        auto& range = line.range();
        if (range.end <= begin or  end < range.begin)
            continue;

        int beg_idx = -1, end_idx = -1;
        for (auto atom_it = line.begin(); atom_it != line.end(); ++atom_it)
        {
            if (not atom_it->has_buffer_range() or
                end <= atom_it->begin() or begin >= atom_it->end())
                continue;

            if (begin >= atom_it->begin())
            {
                if (begin > atom_it->begin())
                    atom_it = ++line.split(atom_it, begin);
                beg_idx = atom_it - line.begin();
            }
            if (end <= atom_it->end())
            {
                if (end < atom_it->end())
                    atom_it = line.split(atom_it, end);
                end_idx = (atom_it - line.begin()) + 1;
            }
        }

        if (beg_idx != -1 and end_idx != -1)
            func(line, beg_idx, end_idx);
    }
}

void apply_highlighter(HighlightContext context,
                       DisplayBuffer& display_buffer,
                       BufferCoord begin, BufferCoord end,
                       Highlighter& highlighter)
{
    if (begin == end)
        return;

    using LineIterator = DisplayBuffer::LineList::iterator;
    LineIterator first_line;
    Vector<DisplayLine::iterator> insert_pos;
    auto line_end = display_buffer.lines().end();

    DisplayBuffer region_display;
    auto& region_lines = region_display.lines();
    for (auto line_it = display_buffer.lines().begin(); line_it != line_end; ++line_it)
    {
        auto& line = *line_it;
        auto& range = line.range();
        if (range.end <= begin or end <= range.begin)
            continue;

        if (region_lines.empty())
            first_line = line_it;
        region_lines.emplace_back();
        insert_pos.emplace_back();

        if (range.begin < begin or range.end > end)
        {
            size_t beg_idx = 0;
            size_t end_idx = line.atoms().size();

            for (auto atom_it = line.begin(); atom_it != line.end(); ++atom_it)
            {
                if (not atom_it->has_buffer_range() or end <= atom_it->begin() or begin >= atom_it->end())
                    continue;

                bool is_replaced = atom_it->type() == DisplayAtom::ReplacedRange;
                if (atom_it->begin() <= begin)
                {
                    if (is_replaced or atom_it->begin() == begin)
                        beg_idx = atom_it - line.begin();
                    else
                    {
                        atom_it = ++line.split(atom_it, begin);
                        beg_idx = atom_it - line.begin();
                        ++end_idx;
                    }
                }

                if (atom_it->end() >= end)
                {
                    if (is_replaced or atom_it->end() == end)
                        end_idx = atom_it - line.begin() + 1;
                    else
                    {
                        atom_it = ++line.split(atom_it, end);
                        end_idx = atom_it - line.begin();
                    }
                }
            }
            std::move(line.begin() + beg_idx, line.begin() + end_idx,
                      std::back_inserter(region_lines.back()));
            insert_pos.back() = line.erase(line.begin() + beg_idx, line.begin() + end_idx);
        }
        else
        {
            region_lines.back() = std::move(line);
            insert_pos.back() = line.begin();
        }
    }

    if (region_display.lines().empty())
        return;

    region_display.compute_range();
    highlighter.highlight(context, region_display, {begin, end});

    for (size_t i = 0; i < region_lines.size(); ++i)
    {
        auto& line = *(first_line + i);
        auto pos = insert_pos[i];
        for (auto& atom : region_lines[i])
            pos = ++line.insert(pos, std::move(atom));
    }
    display_buffer.compute_range();
}

auto apply_face = [](const Face& face)
{
    return [&face](DisplayAtom& atom) {
        atom.face = merge_faces(atom.face, face);
    };
};

static std::unique_ptr<Highlighter> create_fill_highlighter(HighlighterParameters params, Highlighter*)
{
    if (params.size() != 1)
        throw runtime_error("wrong parameter count");

    const String& facespec = params[0];
    auto func = [facespec](HighlightContext context, DisplayBuffer& display_buffer, BufferRange range)
    {
        highlight_range(display_buffer, range.begin, range.end, false,
                        apply_face(context.context.faces()[facespec]));
    };
    return make_highlighter(std::move(func));
}

template<typename T>
struct BufferSideCache
{
    BufferSideCache() : m_id{get_free_value_id()} {}

    T& get(const Buffer& buffer)
    {
        Value& cache_val = buffer.values()[m_id];
        if (not cache_val)
            cache_val = Value(T{});
        return cache_val.as<T>();
    }
private:
    ValueId m_id;
};

using FacesSpec = Vector<std::pair<size_t, String>, MemoryDomain::Highlight>;

class RegexHighlighter : public Highlighter
{
public:
    RegexHighlighter(Regex regex, FacesSpec faces)
        : Highlighter{HighlightPass::Colorize},
          m_regex{std::move(regex)},
          m_faces{std::move(faces)}
    {
        ensure_first_face_is_capture_0();
    }

    void do_highlight(HighlightContext context, DisplayBuffer& display_buffer, BufferRange range) override
    {
        auto overlaps = [](const BufferRange& lhs, const BufferRange& rhs) {
            return lhs.begin < rhs.begin ? lhs.end > rhs.begin
                                         : rhs.end > lhs.begin;
        };

        if (not overlaps(display_buffer.range(), range))
            return;

        const auto faces = m_faces | transform([&faces = context.context.faces()](auto&& spec) {
                return spec.second.empty() ? Face{} : faces[spec.second];
            }) | gather<Vector<Face>>();

        auto& matches = get_matches(context.context.buffer(), display_buffer.range(), range);
        kak_assert(matches.size() % m_faces.size() == 0);
        for (size_t m = 0; m < matches.size(); ++m)
        {
            auto& face = faces[m % faces.size()];
            if (face == Face{})
                continue;

            highlight_range(display_buffer,
                            matches[m].begin, matches[m].end,
                            false, apply_face(face));
        }
    }

    void reset(Regex regex, FacesSpec faces)
    {
        m_regex = std::move(regex);
        m_faces = std::move(faces);
        ensure_first_face_is_capture_0();
        ++m_regex_version;
    }

    static std::unique_ptr<Highlighter> create(HighlighterParameters params, Highlighter*)
    {
        if (params.size() < 2)
            throw runtime_error("wrong parameter count");

        FacesSpec faces;
        for (auto& spec : params.subrange(1))
        {
            auto colon = find(spec, ':');
            if (colon == spec.end())
                throw runtime_error(format("wrong face spec: '{}' expected <capture>:<facespec>", spec));
            int capture = str_to_int({spec.begin(), colon});
            faces.emplace_back(capture, String{colon+1, spec.end()});
        }

        Regex ex{params[0], RegexCompileFlags::Optimize};

        return std::make_unique<RegexHighlighter>(std::move(ex), std::move(faces));
    }

private:
    // stores the range for each highlighted capture of each match
    using MatchList = Vector<BufferRange, MemoryDomain::Highlight>;
    struct Cache
    {
        size_t m_timestamp = -1;
        size_t m_regex_version = -1;
        struct RangeAndMatches { BufferRange range; MatchList matches; };
        Vector<RangeAndMatches, MemoryDomain::Highlight> m_matches;
    };
    BufferSideCache<Cache> m_cache;

    Regex     m_regex;
    FacesSpec m_faces;

    size_t m_regex_version = 0;

    void ensure_first_face_is_capture_0()
    {
        if (m_faces.empty())
            return;

        std::sort(m_faces.begin(), m_faces.end(),
                  [](const std::pair<size_t, String>& lhs,
                     const std::pair<size_t, String>& rhs)
                  { return lhs.first < rhs.first; });
        if (m_faces[0].first != 0)
            m_faces.emplace(m_faces.begin(), 0, String{});
    }

    void add_matches(const Buffer& buffer, MatchList& matches,
                     BufferRange range)
    {
        kak_assert(matches.size() % m_faces.size() == 0);
        using RegexIt = RegexIterator<BufferIterator>;
        RegexIt re_it{get_iterator(buffer, range.begin),
                      get_iterator(buffer, range.end),
                      buffer.begin(), buffer.end(), m_regex,
                      match_flags(is_bol(range.begin),
                                  is_eol(buffer, range.end),
                                  is_bow(buffer, range.begin),
                                  is_eow(buffer, range.end))};
        RegexIt re_end;
        for (; re_it != re_end; ++re_it)
        {
            for (auto& face : m_faces)
            {
                const auto& sub = (*re_it)[face.first];
                matches.push_back({sub.first.coord(), sub.second.coord()});
            }
        }
    }

    MatchList& get_matches(const Buffer& buffer, BufferRange display_range,
                           BufferRange buffer_range)
    {
        Cache& cache = m_cache.get(buffer);
        auto& matches = cache.m_matches;

        if (cache.m_regex_version != m_regex_version or
            cache.m_timestamp != buffer.timestamp())
        {
            matches.clear();
            cache.m_timestamp = buffer.timestamp();
            cache.m_regex_version = m_regex_version;
        }
        const LineCount line_offset = 3;
        BufferRange range{std::max<BufferCoord>(buffer_range.begin, display_range.begin.line - line_offset),
                          std::min<BufferCoord>(buffer_range.end, display_range.end.line + line_offset)};

        auto it = std::upper_bound(matches.begin(), matches.end(), range,
                                   [](const BufferRange& lhs, const Cache::RangeAndMatches& rhs)
                                   { return lhs.begin < rhs.range.end; });

        if (it == matches.end() or it->range.begin > range.end)
        {
            it = matches.insert(it, Cache::RangeAndMatches{range, {}});
            add_matches(buffer, it->matches, range);
        }
        else if (it->matches.empty())
        {
            it->range = range;
            add_matches(buffer, it->matches, range);
        }
        else
        {
            // Here we extend the matches, that is not strictly valid,
            // but may work nicely with every reasonable regex, and
            // greatly reduces regex parsing. To change if we encounter
            // regex that do not work great with that.
            BufferRange& old_range = it->range;
            MatchList& matches = it->matches;

            // Thanks to the ensure_first_face_is_capture_0 method, we know
            // these point to the first/last matches capture 0.
            auto first_end = matches.begin()->end;
            auto last_end = (matches.end() - m_faces.size())->end;

            // add regex matches from new begin to old first match end
            if (range.begin < old_range.begin)
            {
                old_range.begin = range.begin;
                MatchList new_matches;
                add_matches(buffer, new_matches, {range.begin, first_end});
                matches.erase(matches.begin(), matches.begin() + m_faces.size());

                std::copy(std::make_move_iterator(new_matches.begin()),
                          std::make_move_iterator(new_matches.end()),
                          std::inserter(matches, matches.begin()));
            }
            // add regex matches from old last match begin to new end
            if (old_range.end < range.end)
            {
                old_range.end = range.end;
                add_matches(buffer, matches, {last_end, range.end});
            }
        }
        return it->matches;
    }
};

template<typename RegexGetter, typename FaceGetter>
class DynamicRegexHighlighter : public Highlighter
{
public:
    DynamicRegexHighlighter(RegexGetter regex_getter, FaceGetter face_getter)
      : Highlighter{HighlightPass::Colorize},
        m_regex_getter(std::move(regex_getter)),
        m_face_getter(std::move(face_getter)),
        m_highlighter(Regex{}, FacesSpec{}) {}

    void do_highlight(HighlightContext context, DisplayBuffer& display_buffer, BufferRange range) override
    {
        Regex regex = m_regex_getter(context.context);
        FacesSpec face = m_face_getter(context.context);
        if (regex != m_last_regex or face != m_last_face)
        {
            m_last_regex = std::move(regex);
            m_last_face = face;
            if (not m_last_regex.empty())
                m_highlighter.reset(m_last_regex, m_last_face);
        }
        if (not m_last_regex.empty() and not m_last_face.empty())
            m_highlighter.highlight(context, display_buffer, range);
    }

private:
    Regex       m_last_regex;
    RegexGetter m_regex_getter;

    FacesSpec   m_last_face;
    FaceGetter  m_face_getter;

    RegexHighlighter m_highlighter;
};

std::unique_ptr<Highlighter> create_dynamic_regex_highlighter(HighlighterParameters params, Highlighter*)
{
    if (params.size() < 2)
        throw runtime_error("wrong parameter count");

    FacesSpec faces;
    for (auto& spec : params.subrange(1))
    {
        auto colon = find(spec, ':');
        if (colon == spec.end())
            throw runtime_error("wrong face spec: '" + spec +
                                 "' expected <capture>:<facespec>");
        int capture = str_to_int({spec.begin(), colon});
        faces.emplace_back(capture, String{colon+1, spec.end()});
    }


    auto make_hl = [](auto& regex_getter, auto& face_getter) {
        return std::make_unique<DynamicRegexHighlighter<std::decay_t<decltype(regex_getter)>,
                                                        std::decay_t<decltype(face_getter)>>>(
            std::move(regex_getter), std::move(face_getter));
    };
    auto get_face = [faces](const Context& context){ return faces; };

    CommandParser parser{params[0]};
    auto token = parser.read_token(true);
    if (token and parser.done() and token->type == Token::Type::OptionExpand and
        GlobalScope::instance().options()[token->content].is_of_type<Regex>())
    {
        auto get_regex = [option_name = token->content](const Context& context) {
            return context.options()[option_name].get<Regex>();
        };
        return make_hl(get_regex, get_face);
    }

    auto get_regex = [expr = params[0]](const Context& context){
        try
        {
            auto re = expand(expr, context);
            return re.empty() ? Regex{} : Regex{re};
        }
        catch (runtime_error& err)
        {
            write_to_debug_buffer(format("Error while evaluating dynamic regex expression: {}", err.what()));
            return Regex{};
        }
    };
    return make_hl(get_regex, get_face);
}

std::unique_ptr<Highlighter> create_line_highlighter(HighlighterParameters params, Highlighter*)
{
    if (params.size() != 2)
        throw runtime_error("wrong parameter count");

    auto func = [line_expr=params[0], facespec=params[1]]
                (HighlightContext context, DisplayBuffer& display_buffer, BufferRange)
    {
        LineCount line = -1;
        try
        {
            line = str_to_int_ifp(expand(line_expr, context.context)).value_or(0) - 1;
        }
        catch (runtime_error& err)
        {
            write_to_debug_buffer(
                format("Error evaluating highlight line expression: {}", err.what()));
        }

        if (line < 0)
            return;

        auto it = find_if(display_buffer.lines(),
                          [line](const DisplayLine& l)
                          { return l.range().begin.line == line; });
        if (it == display_buffer.lines().end())
            return;

        auto face = context.context.faces()[facespec];
        ColumnCount column = 0;
        for (auto& atom : *it)
        {
            column += atom.length();
            if (!atom.has_buffer_range())
                continue;

            kak_assert(atom.begin().line == line);
            apply_face(face)(atom);
        }
        const ColumnCount remaining = context.context.window().dimensions().column - column;
        if (remaining > 0)
            it->push_back({ String{' ', remaining}, face });
    };

    return make_highlighter(std::move(func));
}

std::unique_ptr<Highlighter> create_column_highlighter(HighlighterParameters params, Highlighter*)
{
    if (params.size() != 2)
        throw runtime_error("wrong parameter count");

    auto func = [col_expr=params[0], facespec=params[1]]
                (HighlightContext context, DisplayBuffer& display_buffer, BufferRange)
    {
        ColumnCount column = -1;
        try
        {
            column = str_to_int_ifp(expand(col_expr, context.context)).value_or(0) - 1;
        }
        catch (runtime_error& err)
        {
            write_to_debug_buffer(
                format("Error evaluating highlight column expression: {}", err.what()));
        }

        if (column < 0)
            return;

        auto face = context.context.faces()[facespec];
        auto win_column = context.setup.window_pos.column;
        for (auto& line : display_buffer.lines())
        {
            auto target_col = column - win_column;
            if (target_col < 0)
                return;

            bool found = false;
            auto first_buf = find_if(line, [](auto& atom) { return atom.has_buffer_range(); });
            for (auto atom_it = first_buf; atom_it != line.end(); ++atom_it)
            {
                const auto atom_len = atom_it->length();
                if (target_col < atom_len)
                {
                    if (target_col > 0)
                        atom_it = ++line.split(atom_it, target_col);
                    if (atom_it->length() > 1)
                        atom_it = line.split(atom_it, 1_col);
                    atom_it->face = merge_faces(atom_it->face, face);
                    found = true;
                    break;
                }
                target_col -= atom_len;
            }
            if (found)
                continue;

            if (target_col > 0)
                line.push_back({String{' ', target_col}, {}});
            line.push_back({" ", face});
        }
    };

    return make_highlighter(std::move(func));
}

struct WrapHighlighter : Highlighter
{
    WrapHighlighter(ColumnCount max_width, bool word_wrap, bool preserve_indent, String marker)
        : Highlighter{HighlightPass::Wrap}, m_max_width{max_width},
          m_word_wrap{word_wrap}, m_preserve_indent{preserve_indent},
          m_marker{std::move(marker)} {}

    static constexpr StringView ms_id = "wrap";

    struct SplitPos{ ByteCount byte; ColumnCount column; };

    void do_highlight(HighlightContext context, DisplayBuffer& display_buffer, BufferRange) override
    {
        if (contains(context.disabled_ids, ms_id))
            return;

        const ColumnCount wrap_column = std::min(m_max_width, context.setup.window_range.column);
        if (wrap_column <= 0)
            return;

        const Buffer& buffer = context.context.buffer();
        const auto& cursor = context.context.selections().main().cursor();
        const int tabstop = context.context.options()["tabstop"].get<int>();
        const LineCount win_height = context.context.window().dimensions().line;
        const ColumnCount marker_len = m_marker.column_length();
        const Face face_marker = context.context.faces()["StatusLineInfo"];
        for (auto it = display_buffer.lines().begin();
             it != display_buffer.lines().end(); ++it)
        {
            const LineCount buf_line = it->range().begin.line;
            const ByteCount line_length = buffer[buf_line].length();
            const ColumnCount indent = m_preserve_indent ? line_indent(buffer, tabstop, buf_line) : 0_col;

            auto pos = next_split_pos(buffer, wrap_column, tabstop, buf_line, {0, 0});
            if (pos.byte == line_length)
                continue;

            for (auto atom_it = it->begin();
                 pos.byte != line_length and atom_it != it->end(); )
            {
                const BufferCoord coord{buf_line, pos.byte};
                if (!atom_it->has_buffer_range() or
                    coord < atom_it->begin() or coord >= atom_it->end())
                {
                    ++atom_it;
                    continue;
                }

                auto& line = *it;

                if (coord > atom_it->begin())
                    atom_it = ++line.split(atom_it, coord);

                DisplayLine new_line{ AtomList{ atom_it, line.end() } };
                line.erase(atom_it, line.end());

                ColumnCount prefix_len = 0;
                if (marker_len != 0 and marker_len < wrap_column)
                {
                    new_line.insert(new_line.begin(), {m_marker, face_marker});
                    prefix_len = marker_len;
                }
                if (indent > marker_len and indent < wrap_column)
                {
                    auto it = new_line.insert(new_line.begin() + (marker_len > 0), {buffer, coord, coord});
                    it->replace(String{' ', indent - marker_len});
                    prefix_len = indent;
                }

                if (it+1 - display_buffer.lines().begin() == win_height)
                {
                    if (cursor >= new_line.range().begin) // strip first lines if cursor is not visible
                    {
                        display_buffer.lines().erase(display_buffer.lines().begin(), display_buffer.lines().begin()+1);
                        --it;
                    }
                    else
                    {
                        display_buffer.lines().erase(it+1, display_buffer.lines().end());
                        return;
                    }
                }
                it = display_buffer.lines().insert(it+1, new_line);

                pos = next_split_pos(buffer, wrap_column - prefix_len, tabstop, buf_line, pos);
                atom_it = it->begin();
            }
        }
    }

    void do_compute_display_setup(HighlightContext context, DisplaySetup& setup) const override
    {
        if (contains(context.disabled_ids, ms_id))
            return;

        const ColumnCount wrap_column = std::min(setup.window_range.column, m_max_width);
        if (wrap_column <= 0)
            return;

        const Buffer& buffer = context.context.buffer();
        const auto& cursor = context.context.selections().main().cursor();
        const int tabstop = context.context.options()["tabstop"].get<int>();

        auto line_wrap_count = [&](LineCount line, ColumnCount indent) {
            LineCount count = 0;
            const ByteCount line_length = buffer[line].length();
            SplitPos pos{0, 0};
            while (true)
            {
                pos = next_split_pos(buffer, wrap_column - (pos.byte == 0 ? 0_col : indent),
                                     tabstop, line, pos);
                if (pos.byte == line_length)
                    break;
                ++count;
            }
            return count;
        };

        const auto win_height = context.context.window().dimensions().line;

        // Disable horizontal scrolling when using a WrapHighlighter
        setup.window_pos.column = 0;
        setup.window_range.line = 0;
        setup.scroll_offset.column = 0;
        setup.full_lines = true;

        const ColumnCount marker_len = m_marker.column_length();

        for (auto buf_line = setup.window_pos.line, win_line = 0_line;
             win_line < win_height or buf_line <= cursor.line;
             ++buf_line, ++setup.window_range.line)
        {
            if (buf_line >= buffer.line_count())
                break;

            const ColumnCount indent = m_preserve_indent ? line_indent(buffer, tabstop, buf_line) : 0_col;

            ColumnCount prefix_len = 0;
            if (marker_len < wrap_column)
                prefix_len = marker_len;
            if (indent > marker_len and indent < wrap_column)
                prefix_len = indent;

            if (buf_line == cursor.line)
            {
                SplitPos pos{0, 0};
                for (LineCount count = 0; true; ++count)
                {
                    auto next_pos = next_split_pos(buffer, wrap_column - (pos.byte != 0 ? prefix_len : 0_col),
                                                   tabstop, buf_line, pos);
                    if (next_pos.byte > cursor.column)
                    {
                        setup.cursor_pos = DisplayCoord{
                            win_line + count,
                            get_column(buffer, tabstop, cursor) -
                            pos.column + (pos.byte != 0 ? indent : 0_col)
                        };
                        break;
                    }
                    pos = next_pos;
                }
                kak_assert(setup.cursor_pos.column >= 0 and setup.cursor_pos.column < setup.window_range.column);
            }
            const auto wrap_count = line_wrap_count(buf_line, prefix_len);
            win_line += wrap_count + 1;

            // scroll window to keep cursor visible, and update range as lines gets removed
            while (buf_line >= cursor.line and setup.window_pos.line < cursor.line and
                   setup.cursor_pos.line + setup.scroll_offset.line >= win_height)
            {
                auto remove_count = std::min(win_height, 1 + line_wrap_count(setup.window_pos.line, indent));
                ++setup.window_pos.line;
                --setup.window_range.line;
                setup.cursor_pos.line -= remove_count;
                win_line -= remove_count;
                kak_assert(setup.cursor_pos.line >= 0);
            }
        }
    }

    void fill_unique_ids(Vector<StringView>& unique_ids) const override
    {
        unique_ids.push_back(ms_id);
    }

    SplitPos next_split_pos(const Buffer& buffer,  ColumnCount wrap_column, int tabstop, LineCount line, SplitPos current) const
    {
        const ColumnCount target_column = current.column + wrap_column;
        StringView content = buffer[line];

        SplitPos pos = current;
        while (pos.byte < content.length() and pos.column < target_column)
        {
            if (content[pos.byte] == '\t')
            {
                const ColumnCount next_column = (pos.column / tabstop + 1) * tabstop;
                if (next_column > target_column and pos.byte != current.byte) // the target column was in the tab
                    break;
                pos.column = next_column;
                ++pos.byte;
            }
            else
            {
                const char* it = &content[pos.byte];
                const ColumnCount width = codepoint_width(utf8::read_codepoint(it, content.end()));
                if (pos.column + width > target_column and pos.byte != current.byte) // the target column was in the char
                    break;
                pos.column += width;
                pos.byte = (int)(it - content.begin());
            }
        }

        if (m_word_wrap and pos.byte < content.length()) // find a word boundary before current position
        {
            utf8::iterator<const char*> it{&content[pos.byte], content};
            while (it != content.begin() and is_word<WORD>(*it))
                --it;

            if (it != content.begin() and it != &content[pos.byte] and
                (it+1) > &content[current.byte])
            {
                const ByteCount word_split = (it+1).base() - content.begin();
                pos.column -= content.substr(word_split, pos.byte - word_split).column_length();
                pos.byte = word_split;
            }
        }
        return pos;
    };

    ColumnCount line_indent(const Buffer& buffer, int tabstop, LineCount line) const
    {
        StringView l = buffer[line];
        auto col = 0_byte;
        while (is_horizontal_blank(l[col]))
               ++col;
        return get_column(buffer, tabstop, {line, col});
    }

    static std::unique_ptr<Highlighter> create(HighlighterParameters params, Highlighter*)
    {
        static const ParameterDesc param_desc{
            { { "word", { false, "" } },
              { "indent", { false, "" } },
              { "width", { true, "" } },
              { "marker", { true, "" } } },
            ParameterDesc::Flags::None, 0, 0
        };<commit_msg>Speed up wrapping at word boundaries.<commit_after>#include "highlighters.hh"

#include "assert.hh"
#include "buffer_utils.hh"
#include "changes.hh"
#include "command_manager.hh"
#include "context.hh"
#include "display_buffer.hh"
#include "face_registry.hh"
#include "highlighter_group.hh"
#include "line_modification.hh"
#include "option.hh"
#include "option_types.hh"
#include "parameters_parser.hh"
#include "ranges.hh"
#include "regex.hh"
#include "register_manager.hh"
#include "string.hh"
#include "utf8.hh"
#include "utf8_iterator.hh"
#include "window.hh"

#include <locale>
#include <cstdio>

namespace Kakoune
{

template<typename Func>
std::unique_ptr<Highlighter> make_highlighter(Func func, HighlightPass pass = HighlightPass::Colorize)
{
    struct SimpleHighlighter : public Highlighter
    {
        SimpleHighlighter(Func func, HighlightPass pass)
          : Highlighter{pass}, m_func{std::move(func)} {}

    private:
        void do_highlight(HighlightContext context, DisplayBuffer& display_buffer, BufferRange range) override
        {
            m_func(context, display_buffer, range);
        }
        Func m_func;
    };
    return std::make_unique<SimpleHighlighter>(std::move(func), pass);
}

template<typename T>
void highlight_range(DisplayBuffer& display_buffer,
                     BufferCoord begin, BufferCoord end,
                     bool skip_replaced, T func)
{
    // tolerate begin > end as that can be triggered by wrong encodings
    if (begin >= end or end <= display_buffer.range().begin
                     or begin >= display_buffer.range().end)
        return;

    for (auto& line : display_buffer.lines())
    {
        auto& range = line.range();
        if (range.end <= begin or  end < range.begin)
            continue;

        for (auto atom_it = line.begin(); atom_it != line.end(); ++atom_it)
        {
            bool is_replaced = atom_it->type() == DisplayAtom::ReplacedRange;

            if (not atom_it->has_buffer_range() or
                (skip_replaced and is_replaced) or
                end <= atom_it->begin() or begin >= atom_it->end())
                continue;

            if (not is_replaced and begin > atom_it->begin())
                atom_it = ++line.split(atom_it, begin);

            if (not is_replaced and end < atom_it->end())
            {
                atom_it = line.split(atom_it, end);
                func(*atom_it);
                ++atom_it;
            }
            else
                func(*atom_it);
        }
    }
}

template<typename T>
void replace_range(DisplayBuffer& display_buffer,
                   BufferCoord begin, BufferCoord end, T func)
{
    // tolerate begin > end as that can be triggered by wrong encodings
    if (begin >= end or end <= display_buffer.range().begin
                     or begin >= display_buffer.range().end)
        return;

    for (auto& line : display_buffer.lines())
    {
        auto& range = line.range();
        if (range.end <= begin or  end < range.begin)
            continue;

        int beg_idx = -1, end_idx = -1;
        for (auto atom_it = line.begin(); atom_it != line.end(); ++atom_it)
        {
            if (not atom_it->has_buffer_range() or
                end <= atom_it->begin() or begin >= atom_it->end())
                continue;

            if (begin >= atom_it->begin())
            {
                if (begin > atom_it->begin())
                    atom_it = ++line.split(atom_it, begin);
                beg_idx = atom_it - line.begin();
            }
            if (end <= atom_it->end())
            {
                if (end < atom_it->end())
                    atom_it = line.split(atom_it, end);
                end_idx = (atom_it - line.begin()) + 1;
            }
        }

        if (beg_idx != -1 and end_idx != -1)
            func(line, beg_idx, end_idx);
    }
}

void apply_highlighter(HighlightContext context,
                       DisplayBuffer& display_buffer,
                       BufferCoord begin, BufferCoord end,
                       Highlighter& highlighter)
{
    if (begin == end)
        return;

    using LineIterator = DisplayBuffer::LineList::iterator;
    LineIterator first_line;
    Vector<DisplayLine::iterator> insert_pos;
    auto line_end = display_buffer.lines().end();

    DisplayBuffer region_display;
    auto& region_lines = region_display.lines();
    for (auto line_it = display_buffer.lines().begin(); line_it != line_end; ++line_it)
    {
        auto& line = *line_it;
        auto& range = line.range();
        if (range.end <= begin or end <= range.begin)
            continue;

        if (region_lines.empty())
            first_line = line_it;
        region_lines.emplace_back();
        insert_pos.emplace_back();

        if (range.begin < begin or range.end > end)
        {
            size_t beg_idx = 0;
            size_t end_idx = line.atoms().size();

            for (auto atom_it = line.begin(); atom_it != line.end(); ++atom_it)
            {
                if (not atom_it->has_buffer_range() or end <= atom_it->begin() or begin >= atom_it->end())
                    continue;

                bool is_replaced = atom_it->type() == DisplayAtom::ReplacedRange;
                if (atom_it->begin() <= begin)
                {
                    if (is_replaced or atom_it->begin() == begin)
                        beg_idx = atom_it - line.begin();
                    else
                    {
                        atom_it = ++line.split(atom_it, begin);
                        beg_idx = atom_it - line.begin();
                        ++end_idx;
                    }
                }

                if (atom_it->end() >= end)
                {
                    if (is_replaced or atom_it->end() == end)
                        end_idx = atom_it - line.begin() + 1;
                    else
                    {
                        atom_it = ++line.split(atom_it, end);
                        end_idx = atom_it - line.begin();
                    }
                }
            }
            std::move(line.begin() + beg_idx, line.begin() + end_idx,
                      std::back_inserter(region_lines.back()));
            insert_pos.back() = line.erase(line.begin() + beg_idx, line.begin() + end_idx);
        }
        else
        {
            region_lines.back() = std::move(line);
            insert_pos.back() = line.begin();
        }
    }

    if (region_display.lines().empty())
        return;

    region_display.compute_range();
    highlighter.highlight(context, region_display, {begin, end});

    for (size_t i = 0; i < region_lines.size(); ++i)
    {
        auto& line = *(first_line + i);
        auto pos = insert_pos[i];
        for (auto& atom : region_lines[i])
            pos = ++line.insert(pos, std::move(atom));
    }
    display_buffer.compute_range();
}

auto apply_face = [](const Face& face)
{
    return [&face](DisplayAtom& atom) {
        atom.face = merge_faces(atom.face, face);
    };
};

static std::unique_ptr<Highlighter> create_fill_highlighter(HighlighterParameters params, Highlighter*)
{
    if (params.size() != 1)
        throw runtime_error("wrong parameter count");

    const String& facespec = params[0];
    auto func = [facespec](HighlightContext context, DisplayBuffer& display_buffer, BufferRange range)
    {
        highlight_range(display_buffer, range.begin, range.end, false,
                        apply_face(context.context.faces()[facespec]));
    };
    return make_highlighter(std::move(func));
}

template<typename T>
struct BufferSideCache
{
    BufferSideCache() : m_id{get_free_value_id()} {}

    T& get(const Buffer& buffer)
    {
        Value& cache_val = buffer.values()[m_id];
        if (not cache_val)
            cache_val = Value(T{});
        return cache_val.as<T>();
    }
private:
    ValueId m_id;
};

using FacesSpec = Vector<std::pair<size_t, String>, MemoryDomain::Highlight>;

class RegexHighlighter : public Highlighter
{
public:
    RegexHighlighter(Regex regex, FacesSpec faces)
        : Highlighter{HighlightPass::Colorize},
          m_regex{std::move(regex)},
          m_faces{std::move(faces)}
    {
        ensure_first_face_is_capture_0();
    }

    void do_highlight(HighlightContext context, DisplayBuffer& display_buffer, BufferRange range) override
    {
        auto overlaps = [](const BufferRange& lhs, const BufferRange& rhs) {
            return lhs.begin < rhs.begin ? lhs.end > rhs.begin
                                         : rhs.end > lhs.begin;
        };

        if (not overlaps(display_buffer.range(), range))
            return;

        const auto faces = m_faces | transform([&faces = context.context.faces()](auto&& spec) {
                return spec.second.empty() ? Face{} : faces[spec.second];
            }) | gather<Vector<Face>>();

        auto& matches = get_matches(context.context.buffer(), display_buffer.range(), range);
        kak_assert(matches.size() % m_faces.size() == 0);
        for (size_t m = 0; m < matches.size(); ++m)
        {
            auto& face = faces[m % faces.size()];
            if (face == Face{})
                continue;

            highlight_range(display_buffer,
                            matches[m].begin, matches[m].end,
                            false, apply_face(face));
        }
    }

    void reset(Regex regex, FacesSpec faces)
    {
        m_regex = std::move(regex);
        m_faces = std::move(faces);
        ensure_first_face_is_capture_0();
        ++m_regex_version;
    }

    static std::unique_ptr<Highlighter> create(HighlighterParameters params, Highlighter*)
    {
        if (params.size() < 2)
            throw runtime_error("wrong parameter count");

        FacesSpec faces;
        for (auto& spec : params.subrange(1))
        {
            auto colon = find(spec, ':');
            if (colon == spec.end())
                throw runtime_error(format("wrong face spec: '{}' expected <capture>:<facespec>", spec));
            int capture = str_to_int({spec.begin(), colon});
            faces.emplace_back(capture, String{colon+1, spec.end()});
        }

        Regex ex{params[0], RegexCompileFlags::Optimize};

        return std::make_unique<RegexHighlighter>(std::move(ex), std::move(faces));
    }

private:
    // stores the range for each highlighted capture of each match
    using MatchList = Vector<BufferRange, MemoryDomain::Highlight>;
    struct Cache
    {
        size_t m_timestamp = -1;
        size_t m_regex_version = -1;
        struct RangeAndMatches { BufferRange range; MatchList matches; };
        Vector<RangeAndMatches, MemoryDomain::Highlight> m_matches;
    };
    BufferSideCache<Cache> m_cache;

    Regex     m_regex;
    FacesSpec m_faces;

    size_t m_regex_version = 0;

    void ensure_first_face_is_capture_0()
    {
        if (m_faces.empty())
            return;

        std::sort(m_faces.begin(), m_faces.end(),
                  [](const std::pair<size_t, String>& lhs,
                     const std::pair<size_t, String>& rhs)
                  { return lhs.first < rhs.first; });
        if (m_faces[0].first != 0)
            m_faces.emplace(m_faces.begin(), 0, String{});
    }

    void add_matches(const Buffer& buffer, MatchList& matches,
                     BufferRange range)
    {
        kak_assert(matches.size() % m_faces.size() == 0);
        using RegexIt = RegexIterator<BufferIterator>;
        RegexIt re_it{get_iterator(buffer, range.begin),
                      get_iterator(buffer, range.end),
                      buffer.begin(), buffer.end(), m_regex,
                      match_flags(is_bol(range.begin),
                                  is_eol(buffer, range.end),
                                  is_bow(buffer, range.begin),
                                  is_eow(buffer, range.end))};
        RegexIt re_end;
        for (; re_it != re_end; ++re_it)
        {
            for (auto& face : m_faces)
            {
                const auto& sub = (*re_it)[face.first];
                matches.push_back({sub.first.coord(), sub.second.coord()});
            }
        }
    }

    MatchList& get_matches(const Buffer& buffer, BufferRange display_range,
                           BufferRange buffer_range)
    {
        Cache& cache = m_cache.get(buffer);
        auto& matches = cache.m_matches;

        if (cache.m_regex_version != m_regex_version or
            cache.m_timestamp != buffer.timestamp())
        {
            matches.clear();
            cache.m_timestamp = buffer.timestamp();
            cache.m_regex_version = m_regex_version;
        }
        const LineCount line_offset = 3;
        BufferRange range{std::max<BufferCoord>(buffer_range.begin, display_range.begin.line - line_offset),
                          std::min<BufferCoord>(buffer_range.end, display_range.end.line + line_offset)};

        auto it = std::upper_bound(matches.begin(), matches.end(), range,
                                   [](const BufferRange& lhs, const Cache::RangeAndMatches& rhs)
                                   { return lhs.begin < rhs.range.end; });

        if (it == matches.end() or it->range.begin > range.end)
        {
            it = matches.insert(it, Cache::RangeAndMatches{range, {}});
            add_matches(buffer, it->matches, range);
        }
        else if (it->matches.empty())
        {
            it->range = range;
            add_matches(buffer, it->matches, range);
        }
        else
        {
            // Here we extend the matches, that is not strictly valid,
            // but may work nicely with every reasonable regex, and
            // greatly reduces regex parsing. To change if we encounter
            // regex that do not work great with that.
            BufferRange& old_range = it->range;
            MatchList& matches = it->matches;

            // Thanks to the ensure_first_face_is_capture_0 method, we know
            // these point to the first/last matches capture 0.
            auto first_end = matches.begin()->end;
            auto last_end = (matches.end() - m_faces.size())->end;

            // add regex matches from new begin to old first match end
            if (range.begin < old_range.begin)
            {
                old_range.begin = range.begin;
                MatchList new_matches;
                add_matches(buffer, new_matches, {range.begin, first_end});
                matches.erase(matches.begin(), matches.begin() + m_faces.size());

                std::copy(std::make_move_iterator(new_matches.begin()),
                          std::make_move_iterator(new_matches.end()),
                          std::inserter(matches, matches.begin()));
            }
            // add regex matches from old last match begin to new end
            if (old_range.end < range.end)
            {
                old_range.end = range.end;
                add_matches(buffer, matches, {last_end, range.end});
            }
        }
        return it->matches;
    }
};

template<typename RegexGetter, typename FaceGetter>
class DynamicRegexHighlighter : public Highlighter
{
public:
    DynamicRegexHighlighter(RegexGetter regex_getter, FaceGetter face_getter)
      : Highlighter{HighlightPass::Colorize},
        m_regex_getter(std::move(regex_getter)),
        m_face_getter(std::move(face_getter)),
        m_highlighter(Regex{}, FacesSpec{}) {}

    void do_highlight(HighlightContext context, DisplayBuffer& display_buffer, BufferRange range) override
    {
        Regex regex = m_regex_getter(context.context);
        FacesSpec face = m_face_getter(context.context);
        if (regex != m_last_regex or face != m_last_face)
        {
            m_last_regex = std::move(regex);
            m_last_face = face;
            if (not m_last_regex.empty())
                m_highlighter.reset(m_last_regex, m_last_face);
        }
        if (not m_last_regex.empty() and not m_last_face.empty())
            m_highlighter.highlight(context, display_buffer, range);
    }

private:
    Regex       m_last_regex;
    RegexGetter m_regex_getter;

    FacesSpec   m_last_face;
    FaceGetter  m_face_getter;

    RegexHighlighter m_highlighter;
};

std::unique_ptr<Highlighter> create_dynamic_regex_highlighter(HighlighterParameters params, Highlighter*)
{
    if (params.size() < 2)
        throw runtime_error("wrong parameter count");

    FacesSpec faces;
    for (auto& spec : params.subrange(1))
    {
        auto colon = find(spec, ':');
        if (colon == spec.end())
            throw runtime_error("wrong face spec: '" + spec +
                                 "' expected <capture>:<facespec>");
        int capture = str_to_int({spec.begin(), colon});
        faces.emplace_back(capture, String{colon+1, spec.end()});
    }


    auto make_hl = [](auto& regex_getter, auto& face_getter) {
        return std::make_unique<DynamicRegexHighlighter<std::decay_t<decltype(regex_getter)>,
                                                        std::decay_t<decltype(face_getter)>>>(
            std::move(regex_getter), std::move(face_getter));
    };
    auto get_face = [faces](const Context& context){ return faces; };

    CommandParser parser{params[0]};
    auto token = parser.read_token(true);
    if (token and parser.done() and token->type == Token::Type::OptionExpand and
        GlobalScope::instance().options()[token->content].is_of_type<Regex>())
    {
        auto get_regex = [option_name = token->content](const Context& context) {
            return context.options()[option_name].get<Regex>();
        };
        return make_hl(get_regex, get_face);
    }

    auto get_regex = [expr = params[0]](const Context& context){
        try
        {
            auto re = expand(expr, context);
            return re.empty() ? Regex{} : Regex{re};
        }
        catch (runtime_error& err)
        {
            write_to_debug_buffer(format("Error while evaluating dynamic regex expression: {}", err.what()));
            return Regex{};
        }
    };
    return make_hl(get_regex, get_face);
}

std::unique_ptr<Highlighter> create_line_highlighter(HighlighterParameters params, Highlighter*)
{
    if (params.size() != 2)
        throw runtime_error("wrong parameter count");

    auto func = [line_expr=params[0], facespec=params[1]]
                (HighlightContext context, DisplayBuffer& display_buffer, BufferRange)
    {
        LineCount line = -1;
        try
        {
            line = str_to_int_ifp(expand(line_expr, context.context)).value_or(0) - 1;
        }
        catch (runtime_error& err)
        {
            write_to_debug_buffer(
                format("Error evaluating highlight line expression: {}", err.what()));
        }

        if (line < 0)
            return;

        auto it = find_if(display_buffer.lines(),
                          [line](const DisplayLine& l)
                          { return l.range().begin.line == line; });
        if (it == display_buffer.lines().end())
            return;

        auto face = context.context.faces()[facespec];
        ColumnCount column = 0;
        for (auto& atom : *it)
        {
            column += atom.length();
            if (!atom.has_buffer_range())
                continue;

            kak_assert(atom.begin().line == line);
            apply_face(face)(atom);
        }
        const ColumnCount remaining = context.context.window().dimensions().column - column;
        if (remaining > 0)
            it->push_back({ String{' ', remaining}, face });
    };

    return make_highlighter(std::move(func));
}

std::unique_ptr<Highlighter> create_column_highlighter(HighlighterParameters params, Highlighter*)
{
    if (params.size() != 2)
        throw runtime_error("wrong parameter count");

    auto func = [col_expr=params[0], facespec=params[1]]
                (HighlightContext context, DisplayBuffer& display_buffer, BufferRange)
    {
        ColumnCount column = -1;
        try
        {
            column = str_to_int_ifp(expand(col_expr, context.context)).value_or(0) - 1;
        }
        catch (runtime_error& err)
        {
            write_to_debug_buffer(
                format("Error evaluating highlight column expression: {}", err.what()));
        }

        if (column < 0)
            return;

        auto face = context.context.faces()[facespec];
        auto win_column = context.setup.window_pos.column;
        for (auto& line : display_buffer.lines())
        {
            auto target_col = column - win_column;
            if (target_col < 0)
                return;

            bool found = false;
            auto first_buf = find_if(line, [](auto& atom) { return atom.has_buffer_range(); });
            for (auto atom_it = first_buf; atom_it != line.end(); ++atom_it)
            {
                const auto atom_len = atom_it->length();
                if (target_col < atom_len)
                {
                    if (target_col > 0)
                        atom_it = ++line.split(atom_it, target_col);
                    if (atom_it->length() > 1)
                        atom_it = line.split(atom_it, 1_col);
                    atom_it->face = merge_faces(atom_it->face, face);
                    found = true;
                    break;
                }
                target_col -= atom_len;
            }
            if (found)
                continue;

            if (target_col > 0)
                line.push_back({String{' ', target_col}, {}});
            line.push_back({" ", face});
        }
    };

    return make_highlighter(std::move(func));
}

struct WrapHighlighter : Highlighter
{
    WrapHighlighter(ColumnCount max_width, bool word_wrap, bool preserve_indent, String marker)
        : Highlighter{HighlightPass::Wrap}, m_max_width{max_width},
          m_word_wrap{word_wrap}, m_preserve_indent{preserve_indent},
          m_marker{std::move(marker)} {}

    static constexpr StringView ms_id = "wrap";

    struct SplitPos{ ByteCount byte; ColumnCount column; };

    void do_highlight(HighlightContext context, DisplayBuffer& display_buffer, BufferRange) override
    {
        if (contains(context.disabled_ids, ms_id))
            return;

        const ColumnCount wrap_column = std::min(m_max_width, context.setup.window_range.column);
        if (wrap_column <= 0)
            return;

        const Buffer& buffer = context.context.buffer();
        const auto& cursor = context.context.selections().main().cursor();
        const int tabstop = context.context.options()["tabstop"].get<int>();
        const LineCount win_height = context.context.window().dimensions().line;
        const ColumnCount marker_len = m_marker.column_length();
        const Face face_marker = context.context.faces()["StatusLineInfo"];
        for (auto it = display_buffer.lines().begin();
             it != display_buffer.lines().end(); ++it)
        {
            const LineCount buf_line = it->range().begin.line;
            const ByteCount line_length = buffer[buf_line].length();
            const ColumnCount indent = m_preserve_indent ? line_indent(buffer, tabstop, buf_line) : 0_col;

            auto pos = next_split_pos(buffer, wrap_column, tabstop, buf_line, {0, 0});
            if (pos.byte == line_length)
                continue;

            for (auto atom_it = it->begin();
                 pos.byte != line_length and atom_it != it->end(); )
            {
                const BufferCoord coord{buf_line, pos.byte};
                if (!atom_it->has_buffer_range() or
                    coord < atom_it->begin() or coord >= atom_it->end())
                {
                    ++atom_it;
                    continue;
                }

                auto& line = *it;

                if (coord > atom_it->begin())
                    atom_it = ++line.split(atom_it, coord);

                DisplayLine new_line{ AtomList{ atom_it, line.end() } };
                line.erase(atom_it, line.end());

                ColumnCount prefix_len = 0;
                if (marker_len != 0 and marker_len < wrap_column)
                {
                    new_line.insert(new_line.begin(), {m_marker, face_marker});
                    prefix_len = marker_len;
                }
                if (indent > marker_len and indent < wrap_column)
                {
                    auto it = new_line.insert(new_line.begin() + (marker_len > 0), {buffer, coord, coord});
                    it->replace(String{' ', indent - marker_len});
                    prefix_len = indent;
                }

                if (it+1 - display_buffer.lines().begin() == win_height)
                {
                    if (cursor >= new_line.range().begin) // strip first lines if cursor is not visible
                    {
                        display_buffer.lines().erase(display_buffer.lines().begin(), display_buffer.lines().begin()+1);
                        --it;
                    }
                    else
                    {
                        display_buffer.lines().erase(it+1, display_buffer.lines().end());
                        return;
                    }
                }
                it = display_buffer.lines().insert(it+1, new_line);

                pos = next_split_pos(buffer, wrap_column - prefix_len, tabstop, buf_line, pos);
                atom_it = it->begin();
            }
        }
    }

    void do_compute_display_setup(HighlightContext context, DisplaySetup& setup) const override
    {
        if (contains(context.disabled_ids, ms_id))
            return;

        const ColumnCount wrap_column = std::min(setup.window_range.column, m_max_width);
        if (wrap_column <= 0)
            return;

        const Buffer& buffer = context.context.buffer();
        const auto& cursor = context.context.selections().main().cursor();
        const int tabstop = context.context.options()["tabstop"].get<int>();

        auto line_wrap_count = [&](LineCount line, ColumnCount indent) {
            LineCount count = 0;
            const ByteCount line_length = buffer[line].length();
            SplitPos pos{0, 0};
            while (true)
            {
                pos = next_split_pos(buffer, wrap_column - (pos.byte == 0 ? 0_col : indent),
                                     tabstop, line, pos);
                if (pos.byte == line_length)
                    break;
                ++count;
            }
            return count;
        };

        const auto win_height = context.context.window().dimensions().line;

        // Disable horizontal scrolling when using a WrapHighlighter
        setup.window_pos.column = 0;
        setup.window_range.line = 0;
        setup.scroll_offset.column = 0;
        setup.full_lines = true;

        const ColumnCount marker_len = m_marker.column_length();

        for (auto buf_line = setup.window_pos.line, win_line = 0_line;
             win_line < win_height or buf_line <= cursor.line;
             ++buf_line, ++setup.window_range.line)
        {
            if (buf_line >= buffer.line_count())
                break;

            const ColumnCount indent = m_preserve_indent ? line_indent(buffer, tabstop, buf_line) : 0_col;

            ColumnCount prefix_len = 0;
            if (marker_len < wrap_column)
                prefix_len = marker_len;
            if (indent > marker_len and indent < wrap_column)
                prefix_len = indent;

            if (buf_line == cursor.line)
            {
                SplitPos pos{0, 0};
                for (LineCount count = 0; true; ++count)
                {
                    auto next_pos = next_split_pos(buffer, wrap_column - (pos.byte != 0 ? prefix_len : 0_col),
                                                   tabstop, buf_line, pos);
                    if (next_pos.byte > cursor.column)
                    {
                        setup.cursor_pos = DisplayCoord{
                            win_line + count,
                            get_column(buffer, tabstop, cursor) -
                            pos.column + (pos.byte != 0 ? indent : 0_col)
                        };
                        break;
                    }
                    pos = next_pos;
                }
                kak_assert(setup.cursor_pos.column >= 0 and setup.cursor_pos.column < setup.window_range.column);
            }
            const auto wrap_count = line_wrap_count(buf_line, prefix_len);
            win_line += wrap_count + 1;

            // scroll window to keep cursor visible, and update range as lines gets removed
            while (buf_line >= cursor.line and setup.window_pos.line < cursor.line and
                   setup.cursor_pos.line + setup.scroll_offset.line >= win_height)
            {
                auto remove_count = std::min(win_height, 1 + line_wrap_count(setup.window_pos.line, indent));
                ++setup.window_pos.line;
                --setup.window_range.line;
                setup.cursor_pos.line -= remove_count;
                win_line -= remove_count;
                kak_assert(setup.cursor_pos.line >= 0);
            }
        }
    }

    void fill_unique_ids(Vector<StringView>& unique_ids) const override
    {
        unique_ids.push_back(ms_id);
    }

    SplitPos next_split_pos(const Buffer& buffer,  ColumnCount wrap_column, int tabstop, LineCount line, SplitPos current) const
    {
        const ColumnCount target_column = current.column + wrap_column;
        StringView content = buffer[line];

        SplitPos pos = current;
        SplitPos last_boundary = {0, 0};

        while (pos.byte < content.length() and pos.column < target_column)
        {
            if (content[pos.byte] == '\t')
            {
                const ColumnCount next_column = (pos.column / tabstop + 1) * tabstop;
                if (next_column > target_column and pos.byte != current.byte) // the target column was in the tab
                    break;
                pos.column = next_column;
                ++pos.byte;
                last_boundary = pos;
            }
            else
            {
                const char* it = &content[pos.byte];
                const Codepoint cp = utf8::read_codepoint(it, content.end());
                const ColumnCount width = codepoint_width(cp);
                if (pos.column + width > target_column and pos.byte != current.byte) // the target column was in the char
                {
                    if (!is_word<WORD>(cp))
                        last_boundary = pos;
                    break;
                }
                pos.column += width;
                pos.byte = (int)(it - content.begin());
                if (!is_word<WORD>(cp))
                    last_boundary = pos;
            }
        }

        if (m_word_wrap and pos.byte < content.length()) // find a word boundary before current position
            if (last_boundary.byte > 0)
                pos = last_boundary;

        return pos;
    };

    ColumnCount line_indent(const Buffer& buffer, int tabstop, LineCount line) const
    {
        StringView l = buffer[line];
        auto col = 0_byte;
        while (is_horizontal_blank(l[col]))
               ++col;
        return get_column(buffer, tabstop, {line, col});
    }

    static std::unique_ptr<Highlighter> create(HighlighterParameters params, Highlighter*)
    {
        static const ParameterDesc param_desc{
            { { "word", { false, "" } },
              { "indent", { false, "" } },
              { "width", { true, "" } },
              { "marker", { true, "" } } },
            ParameterDesc::Flags::None, 0, 0
        };
{'fraction_non_alphanumeric': 0.11476030476551038, 'fraction_numerical': 0.0031233732431025507, 'mean_word_length': 1.822151983261363, 'pattern_counts': {'":': 0, '<': 179, '<?xml version=': 0, '>': 261, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '309857', 'n_tokens_mistral': 18268, 'n_tokens_neox': 17453, 'n_words': 3890}