<commit_before>#include "libride/document.commands.h"

#include <iostream>

#include "base/string.h"
#include "base/utf8.h"
#include "base/c.h"

#include "libride/document.h"
#include "libride/command.h"


namespace
{
    bool is_non_word(char c, const std::string& non_word_chars)
    {
        return non_word_chars.find(c) == std::string::npos;
    }
}

position previous_char(Document* doc, const position& pp);
position next_char(Document* doc, const position& pp);
position previous_word_start(Document* doc, const position& pp, const std::string& non_word_chars);
position next_word_end(Document* doc, const position& pp, const std::string& non_word_chars);
position start_of_word(Document* doc, const position& pp, const std::string& non_word_chars);
position end_of_word(Document* doc, const position& pp, const std::string& non_word_chars);
position previous_block_start(Document* doc, const position& pp);
position next_block_end(Document* doc, const position& pp);
position start_of_line(const position& p);
position end_of_line(const position& p);
position start_of_doc();
position end_of_doc(Document* doc);

position previous_char(Document* doc, const position& pp)
{
    auto p = pp;
    do
    {
        p = doc->position_offset(p, -1);
    }
    while(is_utf8_cont_char(doc->get_char(p)));
    return p;
}


position next_char(Document* doc, const position& pp)
{
    auto p = pp;
    do
    {
        p = doc->position_offset(p, 1);
    } while(is_utf8_cont_char(doc->get_char(p)));
    return p;
}

position previous_word_start_old(Document* doc, const position& pp, const std::string& non_word_chars)
{
    auto p = pp;
    char prev = 0;
    while(p.line > 0 or p.offset > 0)
    {
        const auto np = doc->position_offset(p, -1);
        const auto cchar = doc->get_char(np);
        if( (prev && prev != cchar) || !is_non_word(cchar, non_word_chars))
        {
            break;
        }
        prev = cchar;
        p = np;
    }
    return start_of_word(doc, p, non_word_chars);
}

position previous_word_start(Document* doc, const position& pp, const std::function<bool (char)>& is_word_char)
{
    auto p = pp;

    while(p.line > 0 && p.offset>0 && !is_word_char(doc->get_char(doc->position_offset(p, -1))) )
    {
        p = doc->position_offset(p, -1);
    }

    // skip line if it isn't whitespace
    auto last_p = p;
    while(p.line > 0 && p.offset>0 && is_word_char(doc->get_char(p)) )
    {
        last_p = p;
        p = doc->position_offset(p, -1);
    }

    return last_p;
}


position next_word_end(Document* doc, const position& pp, const std::string& non_word_chars)
{
    auto p = pp;
    char prev = 0;
    const auto end = end_of_doc(doc);
    while(p.line < end.line || p.offset < end.offset)
    {
        const auto cchar = doc->get_char(p);
        if( (prev && prev) != cchar || !is_non_word(cchar, non_word_chars))
        {
            break;
        }
        p = doc->position_offset(p, 1);
        prev = cchar;
    }
    return end_of_word(doc, p, non_word_chars);
}


position start_of_word(Document* doc, const position& pp, const std::string& non_word_chars)
{
    auto p = pp;
    while(true)
    {
        const auto nep = doc->position_offset(p, -1);
        const auto cchar = doc->get_char(nep);
        if( is_non_word(cchar, non_word_chars) || p == nep)
        {
            break;
        }
        p = nep;
    }
    return p;<commit_msg>fix: hopefully fixes compilation issues on CI<commit_after>#include "libride/document.commands.h"

#include <iostream>

#include "base/string.h"
#include "base/utf8.h"
#include "base/c.h"

#include "libride/document.h"
#include "libride/command.h"


namespace
{
    bool is_non_word(char c, const std::string& non_word_chars)
    {
        return non_word_chars.find(c) == std::string::npos;
    }
}

position previous_char(Document* doc, const position& pp);
position next_char(Document* doc, const position& pp);
position previous_word_start(Document* doc, const position& pp, const std::string& non_word_chars);
position next_word_end(Document* doc, const position& pp, const std::string& non_word_chars);
position start_of_word(Document* doc, const position& pp, const std::string& non_word_chars);
position end_of_word(Document* doc, const position& pp, const std::string& non_word_chars);
position previous_block_start(Document* doc, const position& pp);
position next_block_end(Document* doc, const position& pp);
position start_of_line(const position& p);
position end_of_line(const position& p);
position start_of_doc();
position end_of_doc(Document* doc);

position previous_char(Document* doc, const position& pp)
{
    auto p = pp;
    do
    {
        p = doc->position_offset(p, -1);
    }
    while(is_utf8_cont_char(doc->get_char(p)));
    return p;
}


position next_char(Document* doc, const position& pp)
{
    auto p = pp;
    do
    {
        p = doc->position_offset(p, 1);
    } while(is_utf8_cont_char(doc->get_char(p)));
    return p;
}

position previous_word_start_old(Document* doc, const position& pp, const std::string& non_word_chars)
{
    auto p = pp;
    char prev = 0;
    while(p.line > 0 || p.offset > 0)
    {
        const auto np = doc->position_offset(p, -1);
        const auto cchar = doc->get_char(np);
        if( (prev && prev != cchar) || !is_non_word(cchar, non_word_chars))
        {
            break;
        }
        prev = cchar;
        p = np;
    }
    return start_of_word(doc, p, non_word_chars);
}

position previous_word_start(Document* doc, const position& pp, const std::function<bool (char)>& is_word_char)
{
    auto p = pp;

    while(p.line > 0 && p.offset>0 && !is_word_char(doc->get_char(doc->position_offset(p, -1))) )
    {
        p = doc->position_offset(p, -1);
    }

    // skip line if it isn't whitespace
    auto last_p = p;
    while(p.line > 0 && p.offset>0 && is_word_char(doc->get_char(p)) )
    {
        last_p = p;
        p = doc->position_offset(p, -1);
    }

    return last_p;
}


position next_word_end(Document* doc, const position& pp, const std::string& non_word_chars)
{
    auto p = pp;
    char prev = 0;
    const auto end = end_of_doc(doc);
    while(p.line < end.line || p.offset < end.offset)
    {
        const auto cchar = doc->get_char(p);
        if( (prev && prev != cchar) || !is_non_word(cchar, non_word_chars))
        {
            break;
        }
        p = doc->position_offset(p, 1);
        prev = cchar;
    }
    return end_of_word(doc, p, non_word_chars);
}


position start_of_word(Document* doc, const position& pp, const std::string& non_word_chars)
{
    auto p = pp;
    while(true)
    {
        const auto nep = doc->position_offset(p, -1);
        const auto cchar = doc->get_char(nep);
        if( is_non_word(cchar, non_word_chars) || p == nep)
        {
            break;
        }
        p = nep;
    }
    return p;
{'fraction_non_alphanumeric': 0.13231441048034934, 'fraction_numerical': 0.005531295487627365, 'mean_word_length': 3.072910491997629, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5098643', 'n_tokens_mistral': 2424, 'n_tokens_neox': 2400, 'n_words': 582}