<commit_before>#include <QWidget>
#include <QString>
#include <QFont>
#include <QMainWindow>
#include <QVBoxLayout>
#include <QApplication>
#include <QEvent>
#include "textedit.h"

// ### TODO ###
// ### Should clear selection when something else selects something.
// ### Line break. Could vastly simplify textlayout if not breaking lines.
// ### saving to same file. Need something there.
// ### could refactor chunks so that I only have one and split when I need. Not sure if it's worth it. Would
// ### need chunkData(int from = 0, int size = -1). The current approach makes caching easier since I can now cache an entire chunk.
// ### ensureCursorVisible(TextCursor) is not implemented
// ### add a command line switch to randomly do stuff. Insert, move around etc n times to see if I can reproduce a crash. Allow input of seed
// ### I still have some weird debris when scrolling in large documents
// ### use tests from Qt. E.g. for QTextCursor
// ### need to protect against undo with huge amounts of text in it. E.g. select all and delete. MsgBox asking?
// ### maybe refactor updatePosition so it can handle the margins usecase that textcursor needs.
// ### could keep undo/redo history in the textcursor and have an api on the cursor.
// ### block state in SyntaxHighlighter doesn't remember between
// ### highlights. In regular QText.* it does. Could store this info.
// ### Shouldn't be that much. Maybe base it off of position in
// ### document rather than line number since I don't know which line
// ### we're on
// ### TextCursor(const TextEdit *). Is this a good idea?
// ### caching stuff is getting a little convoluted
// ### what should TextDocument::read(documentSize + 1, 10) do? ASSERT?
// ### should I allow to write in a formatted manner? currentTextFormat and all? Why not really.
// ### consider using QTextBoundaryFinder for something
// ### drag and drop
// ### documentation
// ### consider having extra textLayouts on each side of viewport for optimized scrolling. Could detect that condition
// ### Undo section removal/adding. This is a mess


class Highlighter : public SyntaxHighlighter
{
public:
    Highlighter(QWidget *parent)
        : SyntaxHighlighter(parent)
    {

    }

    virtual void highlightBlock(const QString &text)
    {
        QTextBlockFormat format;
        if (previousBlockState() == 1) {
            format.setBackground(Qt::yellow);
            setCurrentBlockState(0);
        } else {
            format.setBackground(Qt::lightGray);
            format.setLeftMargin(20);
            setCurrentBlockState(1);
        }
        setBlockFormat(format);
    }
};

bool add = false;
class MainWindow : public QMainWindow
{
    Q_OBJECT
public:
    MainWindow(QWidget *parent = 0)
        : QMainWindow(parent), doLineNumbers(false)
    {
        QString fileName = "main.cpp";
        bool replay = false;
        bool readOnly = false;
        int chunkSize = -1;
        QTextCodec *codec = 0;
        const QStringList list = QApplication::arguments().mid(1);
        for (int i=0; i<list.size(); ++i) {
            const QString &arg = list.at(i);
            if (arg == "--replay") {
                replay = true;
                fileName.clear();<commit_msg>more awesome highlighting in example<commit_after>#include <QWidget>
#include <QString>
#include <QFont>
#include <QMainWindow>
#include <QVBoxLayout>
#include <QApplication>
#include <QEvent>
#include "textedit.h"

// ### TODO ###
// ### Should clear selection when something else selects something.
// ### Line break. Could vastly simplify textlayout if not breaking lines.
// ### saving to same file. Need something there.
// ### could refactor chunks so that I only have one and split when I need. Not sure if it's worth it. Would
// ### need chunkData(int from = 0, int size = -1). The current approach makes caching easier since I can now cache an entire chunk.
// ### ensureCursorVisible(TextCursor) is not implemented
// ### add a command line switch to randomly do stuff. Insert, move around etc n times to see if I can reproduce a crash. Allow input of seed
// ### I still have some weird debris when scrolling in large documents
// ### use tests from Qt. E.g. for QTextCursor
// ### need to protect against undo with huge amounts of text in it. E.g. select all and delete. MsgBox asking?
// ### maybe refactor updatePosition so it can handle the margins usecase that textcursor needs.
// ### could keep undo/redo history in the textcursor and have an api on the cursor.
// ### block state in SyntaxHighlighter doesn't remember between
// ### highlights. In regular QText.* it does. Could store this info.
// ### Shouldn't be that much. Maybe base it off of position in
// ### document rather than line number since I don't know which line
// ### we're on
// ### TextCursor(const TextEdit *). Is this a good idea?
// ### caching stuff is getting a little convoluted
// ### what should TextDocument::read(documentSize + 1, 10) do? ASSERT?
// ### should I allow to write in a formatted manner? currentTextFormat and all? Why not really.
// ### consider using QTextBoundaryFinder for something
// ### drag and drop
// ### documentation
// ### consider having extra textLayouts on each side of viewport for optimized scrolling. Could detect that condition
// ### Undo section removal/adding. This is a mess


class Highlighter : public SyntaxHighlighter
{
public:
    Highlighter(QWidget *parent)
        : SyntaxHighlighter(parent)
    {

    }

    void helper(int from, int size, bool blackForeground)
    {
        QTextCharFormat format;
        format.setBackground(blackForeground ? Qt::yellow : Qt::black);
        format.setForeground(blackForeground ? Qt::black : Qt::yellow);
        setFormat(from, size, format);
    }

    virtual void highlightBlock(const QString &text)
    {
        enum { Space, LetterOrNumber, Other } state = Space;
        int last = 0;
        for (int i=0; i<text.size(); ++i) {
            if (text.at(i).isSpace()) {
                if (state != Space)
                    helper(last, i - last, state == LetterOrNumber);
                state = Space;
                last = i;
            } else if (text.at(i).isLetterOrNumber()
                       != (state == LetterOrNumber) || state == Space) {
                if (state != Space) {
                    helper(last, i - last, state == LetterOrNumber);
                }
                state = (text.at(i).isLetterOrNumber()
                         ? LetterOrNumber
                         : Other);
                last = i;
            }
        }
        if (state != Space) {
            helper(last, text.size() - last, state == LetterOrNumber);
        }
    }
};

bool add = false;
class MainWindow : public QMainWindow
{
    Q_OBJECT
public:
    MainWindow(QWidget *parent = 0)
        : QMainWindow(parent), doLineNumbers(false)
    {
        QString fileName = "main.cpp";
        bool replay = false;
        bool readOnly = false;
        int chunkSize = -1;
        QTextCodec *codec = 0;
        const QStringList list = QApplication::arguments().mid(1);
        for (int i=0; i<list.size(); ++i) {
            const QString &arg = list.at(i);
            if (arg == "--replay") {
                replay = true;
                fileName.clear();
{'fraction_non_alphanumeric': 0.1118421052631579, 'fraction_numerical': 0.003700657894736842, 'mean_word_length': 2.6267395626242545, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6897167', 'n_tokens_mistral': 2008, 'n_tokens_neox': 1993, 'n_words': 788}