<commit_before>/**************************************************************************
**
** This file is part of Qt Creator
**
** Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
**
** Contact: Nokia Corporation (<EMAIL>)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/

#include <qmljs/qmljshighlighter.h>

#include <QtCore/QSet>
#include <QtCore/QtAlgorithms>
#include <QtCore/QDebug>

using namespace QmlJS;

QScriptHighlighter::QScriptHighlighter(bool duiEnabled, QTextDocument *parent):
        QSyntaxHighlighter(parent),
        m_duiEnabled(duiEnabled)
{
    QVector<QTextCharFormat> rc;
    rc.resize(NumFormats);
    rc[NumberFormat].setForeground(Qt::blue);
    rc[StringFormat].setForeground(Qt::darkGreen);
    rc[TypeFormat].setForeground(Qt::darkMagenta);
    rc[KeywordFormat].setForeground(Qt::darkYellow);
    rc[LabelFormat].setForeground(Qt::darkRed);
    rc[CommentFormat].setForeground(Qt::red); rc[CommentFormat].setFontItalic(true);
    rc[PreProcessorFormat].setForeground(Qt::darkBlue);
    rc[VisualWhitespace].setForeground(Qt::lightGray); // for debug: rc[VisualWhitespace].setBackground(Qt::red);
    setFormats(rc);

    m_scanner.setKeywords(keywords());
}

bool QScriptHighlighter::isDuiEnabled() const
{ return m_duiEnabled; }

static bool checkStartOfBinding(const Token &token)
{
    switch (token.kind) {
    case Token::Semicolon:
    case Token::LeftBrace:
    case Token::RightBrace:
    case Token::LeftBracket:
    case Token::RightBracket:
        return true;

    default:
        return false;
    } // end of switch
}

void QScriptHighlighter::highlightBlock(const QString &text)
{
    const QList<Token> tokens = m_scanner(text, onBlockStart());

    int index = 0;
    while (index < tokens.size()) {
        const Token &token = tokens.at(index);

        switch (token.kind) {
            case Token::Keyword:
                setFormat(token.offset, token.length, m_formats[KeywordFormat]);
                break;

            case Token::String:
                highlightWhitespace(token, text, StringFormat);
                break;

            case Token::Comment:
                highlightWhitespace(token, text, CommentFormat);
                break;

            case Token::Number:
                highlightWhitespace(token, text, NumberFormat);
                break;

            case Token::LeftParenthesis:
                onOpeningParenthesis('(', token.offset);
                break;

            case Token::RightParenthesis:
                onClosingParenthesis(')', token.offset);
                break;

            case Token::LeftBrace:
                onOpeningParenthesis('{', token.offset);
                break;

            case Token::RightBrace:
                onClosingParenthesis('}', token.offset);
                break;

            case Token::LeftBracket:
                onOpeningParenthesis('[', token.offset);
                break;

            case Token::RightBracket:
                onClosingParenthesis(']', token.offset);
                break;

            case Token::Identifier: {
                if (index + 1 < tokens.size()) {
                    if (tokens.at(index + 1).is(Token::LeftBrace) && text.at(token.offset).isUpper()) {
                        setFormat(token.offset, token.length, m_formats[TypeFormat]);
                    } else if (index == 0 || checkStartOfBinding(tokens.at(index - 1))) {
                        const int start = index;

                        ++index; // skip the identifier.
                        while (index + 1 < tokens.size() &&
                               tokens.at(index).is(Token::Dot) &&
                               tokens.at(index + 1).is(Token::Identifier)) {
                            index += 2;
                        }

                        if (index < tokens.size() && tokens.at(index).is(Token::Colon)) {
                            // it's a binding.
                            for (int i = start; i < index; ++i) {
                                const Token &tok = tokens.at(i);
                                setFormat(tok.offset, tok.length, m_formats[LabelFormat]);
                            }
                            break;
                        } else {
                            index = start;
                        }
                    }
                }
            }   break;

            case Token::Delimiter:
                break;

            default:
                break;
        } // end swtich

        ++index;
    }

    int firstNonSpace = 0;

    if (! tokens.isEmpty()) {
        const Token &tk = tokens.first();
        firstNonSpace = tk.offset;
    }

    setCurrentBlockState(m_scanner.endState());
    onBlockEnd(m_scanner.endState(), firstNonSpace);
}

void QScriptHighlighter::setFormats(const QVector<QTextCharFormat> &s)
{
    Q_ASSERT(s.size() == NumFormats);
    qCopy(s.constBegin(), s.constEnd(), m_formats);
}

QSet<QString> QScriptHighlighter::keywords()
{
    QSet<QString> keywords;
<commit_msg>Mark the white spaces using the VisualWhitespace format.<commit_after>/**************************************************************************
**
** This file is part of Qt Creator
**
** Copyright (c) 2009 Nokia Corporation and/or its subsidiary(-ies).
**
** Contact: Nokia Corporation (<EMAIL>)
**
** Commercial Usage
**
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
**
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** If you are unsure which license is appropriate for your use, please
** contact the sales department at http://qt.nokia.com/contact.
**
**************************************************************************/

#include <qmljs/qmljshighlighter.h>

#include <QtCore/QSet>
#include <QtCore/QtAlgorithms>
#include <QtCore/QDebug>

using namespace QmlJS;

QScriptHighlighter::QScriptHighlighter(bool duiEnabled, QTextDocument *parent):
        QSyntaxHighlighter(parent),
        m_duiEnabled(duiEnabled)
{
    QVector<QTextCharFormat> rc;
    rc.resize(NumFormats);
    rc[NumberFormat].setForeground(Qt::blue);
    rc[StringFormat].setForeground(Qt::darkGreen);
    rc[TypeFormat].setForeground(Qt::darkMagenta);
    rc[KeywordFormat].setForeground(Qt::darkYellow);
    rc[LabelFormat].setForeground(Qt::darkRed);
    rc[CommentFormat].setForeground(Qt::red); rc[CommentFormat].setFontItalic(true);
    rc[PreProcessorFormat].setForeground(Qt::darkBlue);
    rc[VisualWhitespace].setForeground(Qt::lightGray); // for debug: rc[VisualWhitespace].setBackground(Qt::red);
    setFormats(rc);

    m_scanner.setKeywords(keywords());
}

bool QScriptHighlighter::isDuiEnabled() const
{ return m_duiEnabled; }

static bool checkStartOfBinding(const Token &token)
{
    switch (token.kind) {
    case Token::Semicolon:
    case Token::LeftBrace:
    case Token::RightBrace:
    case Token::LeftBracket:
    case Token::RightBracket:
        return true;

    default:
        return false;
    } // end of switch
}

void QScriptHighlighter::highlightBlock(const QString &text)
{
    const QList<Token> tokens = m_scanner(text, onBlockStart());

    int index = 0;
    while (index < tokens.size()) {
        const Token &token = tokens.at(index);

        switch (token.kind) {
            case Token::Keyword:
                setFormat(token.offset, token.length, m_formats[KeywordFormat]);
                break;

            case Token::String:
                highlightWhitespace(token, text, StringFormat);
                break;

            case Token::Comment:
                highlightWhitespace(token, text, CommentFormat);
                break;

            case Token::Number:
                highlightWhitespace(token, text, NumberFormat);
                break;

            case Token::LeftParenthesis:
                onOpeningParenthesis('(', token.offset);
                break;

            case Token::RightParenthesis:
                onClosingParenthesis(')', token.offset);
                break;

            case Token::LeftBrace:
                onOpeningParenthesis('{', token.offset);
                break;

            case Token::RightBrace:
                onClosingParenthesis('}', token.offset);
                break;

            case Token::LeftBracket:
                onOpeningParenthesis('[', token.offset);
                break;

            case Token::RightBracket:
                onClosingParenthesis(']', token.offset);
                break;

            case Token::Identifier: {
                if (index + 1 < tokens.size()) {
                    if (tokens.at(index + 1).is(Token::LeftBrace) && text.at(token.offset).isUpper()) {
                        setFormat(token.offset, token.length, m_formats[TypeFormat]);
                    } else if (index == 0 || checkStartOfBinding(tokens.at(index - 1))) {
                        const int start = index;

                        ++index; // skip the identifier.
                        while (index + 1 < tokens.size() &&
                               tokens.at(index).is(Token::Dot) &&
                               tokens.at(index + 1).is(Token::Identifier)) {
                            index += 2;
                        }

                        if (index < tokens.size() && tokens.at(index).is(Token::Colon)) {
                            // it's a binding.
                            for (int i = start; i < index; ++i) {
                                const Token &tok = tokens.at(i);
                                setFormat(tok.offset, tok.length, m_formats[LabelFormat]);
                            }
                            break;
                        } else {
                            index = start;
                        }
                    }
                }
            }   break;

            case Token::Delimiter:
                break;

            default:
                break;
        } // end swtich

        ++index;
    }

    int previousTokenEnd = 0;
    for (int i = 0; i < tokens.size(); ++i) {
        const Token &token = tokens.at(i);
        setFormat(previousTokenEnd, token.begin() - previousTokenEnd, m_formats[VisualWhitespace]);
        previousTokenEnd = token.end();
    }

    int firstNonSpace = 0;
    if (! tokens.isEmpty()) {
        const Token &tk = tokens.first();
        firstNonSpace = tk.offset;
    }

    setCurrentBlockState(m_scanner.endState());
    onBlockEnd(m_scanner.endState(), firstNonSpace);
}

void QScriptHighlighter::setFormats(const QVector<QTextCharFormat> &s)
{
    Q_ASSERT(s.size() == NumFormats);
    qCopy(s.constBegin(), s.constEnd(), m_formats);
}

QSet<QString> QScriptHighlighter::keywords()
{
    QSet<QString> keywords;

{'fraction_non_alphanumeric': 0.13269293389775316, 'fraction_numerical': 0.0032562683165092803, 'mean_word_length': 2.148385443362378, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '912103', 'n_tokens_mistral': 3401, 'n_tokens_neox': 3208, 'n_words': 783}