<commit_before>///////////////////////////////////////////////////////////////////////////////
//
// mSIGNA
//
// wordlistvalidator.cpp
//
// Copyright (c) 2015 <NAME>
//
// All Rights Reserved.

#include "wordlistvalidator.h"

#include <string>

WordlistValidator::WordlistValidator(int minlen, int maxlen, QObject* parent) :
    QValidator(parent), minlen_(minlen), maxlen_(maxlen)
{
}

QValidator::State WordlistValidator::validate(QString& input, int& pos) const
{
    if (input.isEmpty()) return Intermediate;
    input = input.toLower();

    // Backspace over spaces if we're not at the end.
    if ((pos < input.size()) && (input.size() + 1 == lastInput.size()) && (lastInput[pos] == ' '))
    {
        QString temp = input.left(pos) + ' ' + input.right(input.size() - pos);
        if (temp == lastInput) input = lastInput;
    }

    QString newInput;
    int newPos = pos;
    
    std::string wordlist = input.toStdString();
    int len = wordlist.size();

    bool removeSpaces = true;
    bool final = true;
    QString lastWord;
    int i = 0;
    for (auto c: wordlist)
    {
        i++;
        if (c == ' ')
        {
            if (lastWord.isEmpty())
            {
                // Strip extra spaces
                if (newPos > newInput.size()) newPos--;
            }
            else if (lastWord.size() >= minlen_)
            {
                newInput += lastWord + ' ';
                lastWord.clear();
            }
            else if (i < wordlist.size() && newPos > newInput.size())
            {
                // Remove words that are too short.
                newPos = newInput.size() - 1;
                lastWord.clear();
            }
            else
            {
                // Disallow adding a space that makes a word too short.<commit_msg>Removed unused variables from WordlistValidator::validate().<commit_after>///////////////////////////////////////////////////////////////////////////////
//
// mSIGNA
//
// wordlistvalidator.cpp
//
// Copyright (c) 2015 <NAME>
//
// All Rights Reserved.

#include "wordlistvalidator.h"

#include <string>

WordlistValidator::WordlistValidator(int minlen, int maxlen, QObject* parent) :
    QValidator(parent), minlen_(minlen), maxlen_(maxlen)
{
}

QValidator::State WordlistValidator::validate(QString& input, int& pos) const
{
    if (input.isEmpty()) return Intermediate;
    input = input.toLower();

    // Backspace over spaces if we're not at the end.
    if ((pos < input.size()) && (input.size() + 1 == lastInput.size()) && (lastInput[pos] == ' '))
    {
        QString temp = input.left(pos) + ' ' + input.right(input.size() - pos);
        if (temp == lastInput) input = lastInput;
    }

    QString newInput;
    int newPos = pos;
    
    std::string wordlist = input.toStdString();
    bool final = true;
    QString lastWord;
    int i = 0;
    for (auto c: wordlist)
    {
        i++;
        if (c == ' ')
        {
            if (lastWord.isEmpty())
            {
                // Strip extra spaces
                if (newPos > newInput.size()) newPos--;
            }
            else if (lastWord.size() >= minlen_)
            {
                newInput += lastWord + ' ';
                lastWord.clear();
            }
            else if (i < wordlist.size() && newPos > newInput.size())
            {
                // Remove words that are too short.
                newPos = newInput.size() - 1;
                lastWord.clear();
            }
            else
            {
                // Disallow adding a space that makes a word too short.
{'fraction_non_alphanumeric': 0.17454545454545456, 'fraction_numerical': 0.003916083916083916, 'mean_word_length': 1.550641940085592, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5598563', 'n_tokens_mistral': 1028, 'n_tokens_neox': 998, 'n_words': 263}