<commit_before>#ifndef CTOOL_BASE
#define CTOOL_BASE

#include <string>
#include <vector>
#include <list>
#include <stdio.h>
#include <string.h>

#ifndef _WIN32
#include <unistd.h>
#include <termios.h>
#else
#include <conio.h>
#endif

#include "hecl/Database.hpp"
#include "logvisor/logvisor.hpp"

extern logvisor::Module LogModule;

struct ToolPassInfo
{
    hecl::SystemString pname;
    hecl::SystemString cwd;
    std::vector<hecl::SystemString> args;
    std::vector<hecl::SystemChar> flags;
    hecl::SystemString output;
    hecl::Database::Project* project = nullptr;
    unsigned verbosityLevel = 0;
    bool force = false;
    bool yes = false;
};

#define RED "\033[0;31m"
#define GREEN "\033[0;32m"
#define YELLOW "\033[0;33m"
#define BLUE "\033[0;34m"
#define MAGENTA "\033[0;35m"
#define CYAN "\033[0;36m"
#define BOLD "\033[1m"
#define NORMAL "\033[0m"

#define HIDE_CURSOR "\033[?25l"
#define SHOW_CURSOR "\033[?25h"

#define WRAP_INDENT 4

extern bool XTERM_COLOR;

class ToolBase
{
protected:
    const ToolPassInfo& m_info;
    bool m_good = false;

    bool continuePrompt()
    {
        if (!m_info.yes)
        {
            if (XTERM_COLOR)
                hecl::Printf(_S("\n" BLUE BOLD "Continue?" NORMAL " (Y/n) "));
            else
                hecl::Printf(_S("\nContinue? (Y/n) "));

            int ch;
#ifndef _WIN32
            struct termios tioOld, tioNew;
            tcgetattr(0, &tioOld);
            tioNew = tioOld;
            tioNew.c_lflag &= ~ICANON;
            tcsetattr(0, TCSANOW, &tioNew);
            while ((ch = getchar()))
#else
            while ((ch = getch()))
#endif
            {
                if (ch == 'n' || ch == 'N')
                    return false;
                if (ch == 'y' || ch == 'Y' || ch == '\r' || ch == '\n')
                    break;
            }
#ifndef _WIN32
            tcsetattr(0, TCSANOW, &tioOld);
#endif
        }
        hecl::Printf(_S("\n"));
        return true;
    }

public:
    ToolBase(const ToolPassInfo& info)
    : m_info(info)
    {
        hecl::VerbosityLevel = info.verbosityLevel;
    }
    virtual ~ToolBase() {}
    virtual hecl::SystemString toolName() const=0;
    virtual int run()=0;
    inline operator bool() const {return m_good;}
};

class HelpOutput
{
public:
    typedef void(*HelpFunc)(HelpOutput&);
private:
    FILE* m_sout;
    HelpFunc m_helpFunc;
    int m_lineWidth;
    hecl::SystemString m_wrapBuffer;

    void _wrapBuf(hecl::SystemString& string)
    {
        int counter;
        hecl::SystemString::iterator it = string.begin();

        while (it != string.end())
        {
            hecl::SystemString::iterator v=it;

            /* copy string until the end of the line is reached */
            for (counter=WRAP_INDENT ; counter < m_lineWidth ; ++counter)
            {
                if (*it == _S('\n'))
                {
                    counter = WRAP_INDENT;
                    ++it;
                }
                if (counter == WRAP_INDENT)
                {
                    for (int i=0 ; i<WRAP_INDENT ; ++i)
                        it = string.insert(it, _S(' ')) + 1;
                }
                if (it >= string.end())
                    return;
                if (*it != _S('\n'))
                    ++it;
            }
            /* check for whitespace */
            if (isspace(*it))
            {
                *it = _S('\n');
                counter = WRAP_INDENT;
                ++it;
            }
            else
            {
                /* check for nearest whitespace back in string */
                for (hecl::SystemString::iterator k=it ; k!=string.begin() ; --k)
                {
                    if (isspace(*k))
                    {
                        counter = WRAP_INDENT;
                        if (k < v)
                        {
                            k = it;
                            string.insert(k, _S('\n'));
                        }
                        else
                            *k = _S('\n');
                        it = k + 1;
                        break;
                    }
                }
            }
        }
    }

public:

    HelpOutput(HelpFunc helpFunc)
    : m_sout(NULL), m_helpFunc(helpFunc), m_lineWidth(hecl::ConsoleWidth())
    {}
<commit_msg>Fix help printout iterator edge cases<commit_after>#ifndef CTOOL_BASE
#define CTOOL_BASE

#include <string>
#include <vector>
#include <list>
#include <stdio.h>
#include <string.h>

#ifndef _WIN32
#include <unistd.h>
#include <termios.h>
#else
#include <conio.h>
#endif

#include "hecl/Database.hpp"
#include "logvisor/logvisor.hpp"

extern logvisor::Module LogModule;

struct ToolPassInfo
{
    hecl::SystemString pname;
    hecl::SystemString cwd;
    std::vector<hecl::SystemString> args;
    std::vector<hecl::SystemChar> flags;
    hecl::SystemString output;
    hecl::Database::Project* project = nullptr;
    unsigned verbosityLevel = 0;
    bool force = false;
    bool yes = false;
};

#define RED "\033[0;31m"
#define GREEN "\033[0;32m"
#define YELLOW "\033[0;33m"
#define BLUE "\033[0;34m"
#define MAGENTA "\033[0;35m"
#define CYAN "\033[0;36m"
#define BOLD "\033[1m"
#define NORMAL "\033[0m"

#define HIDE_CURSOR "\033[?25l"
#define SHOW_CURSOR "\033[?25h"

#define WRAP_INDENT 4

extern bool XTERM_COLOR;

class ToolBase
{
protected:
    const ToolPassInfo& m_info;
    bool m_good = false;

    bool continuePrompt()
    {
        if (!m_info.yes)
        {
            if (XTERM_COLOR)
                hecl::Printf(_S("\n" BLUE BOLD "Continue?" NORMAL " (Y/n) "));
            else
                hecl::Printf(_S("\nContinue? (Y/n) "));

            int ch;
#ifndef _WIN32
            struct termios tioOld, tioNew;
            tcgetattr(0, &tioOld);
            tioNew = tioOld;
            tioNew.c_lflag &= ~ICANON;
            tcsetattr(0, TCSANOW, &tioNew);
            while ((ch = getchar()))
#else
            while ((ch = getch()))
#endif
            {
                if (ch == 'n' || ch == 'N')
                    return false;
                if (ch == 'y' || ch == 'Y' || ch == '\r' || ch == '\n')
                    break;
            }
#ifndef _WIN32
            tcsetattr(0, TCSANOW, &tioOld);
#endif
        }
        hecl::Printf(_S("\n"));
        return true;
    }

public:
    ToolBase(const ToolPassInfo& info)
    : m_info(info)
    {
        hecl::VerbosityLevel = info.verbosityLevel;
    }
    virtual ~ToolBase() {}
    virtual hecl::SystemString toolName() const=0;
    virtual int run()=0;
    inline operator bool() const {return m_good;}
};

class HelpOutput
{
public:
    typedef void(*HelpFunc)(HelpOutput&);
private:
    FILE* m_sout;
    HelpFunc m_helpFunc;
    int m_lineWidth;
    hecl::SystemString m_wrapBuffer;

    void _wrapBuf(hecl::SystemString& string)
    {
        int counter;
        hecl::SystemString::iterator it = string.begin();

        while (it != string.end())
        {
            std::ptrdiff_t v = it - string.begin();

            /* copy string until the end of the line is reached */
            for (counter=WRAP_INDENT ; counter < m_lineWidth ; ++counter)
            {
                if (it >= string.end())
                    return;
                if (*it == _S('\n'))
                {
                    counter = WRAP_INDENT;
                    ++it;
                }
                if (counter == WRAP_INDENT)
                {
                    for (int i=0 ; i<WRAP_INDENT ; ++i)
                        it = string.insert(it, _S(' ')) + 1;
                }
                if (it >= string.end())
                    return;
                if (*it != _S('\n'))
                    ++it;
            }
            /* check for whitespace */
            if (isspace(*it))
            {
                *it = _S('\n');
                counter = WRAP_INDENT;
                ++it;
            }
            else
            {
                /* check for nearest whitespace back in string */
                for (hecl::SystemString::iterator k=it ; k!=string.begin() ; --k)
                {
                    if (isspace(*k))
                    {
                        counter = WRAP_INDENT;
                        if (k - string.begin() < v)
                            k = string.insert(it, _S('\n'));
                        else
                            *k = _S('\n');
                        it = k + 1;
                        break;
                    }
                }
            }
        }
    }

public:

    HelpOutput(HelpFunc helpFunc)
    : m_sout(NULL), m_helpFunc(helpFunc), m_lineWidth(hecl::ConsoleWidth())
    {}

{'fraction_non_alphanumeric': 0.13135739822396494, 'fraction_numerical': 0.016145773267212546, 'mean_word_length': 1.4770065695515566, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5252401', 'n_tokens_mistral': 2864, 'n_tokens_neox': 2699, 'n_words': 617}