<commit_before>//
//  CInputEnglish.cpp
//  Words
//
//  Created by <NAME> on 2014-03-03.
//  Copyright (c) 2014 Groarke & Co. All rights reserved.
//

#include "CInputEnglish.h"

CInputEnglish::CInputEnglish(CMap &map)
{
    _map = &map;
    _markovLength = 3;
}


void CInputEnglish::ReadStdin()
{

}


void CInputEnglish::ReadFile(string filename)
{

    ifstream file;
    file.open( filename.c_str(), ifstream::in );
    string word;
    word.clear();

    while( file.good() )
    {
        char c = file.get();   // get character from file

        if( file.good() )
        {
            switch( c )
            {
                case ' ':
//                    cout << word << endl;
                    addWord(word);
                    break;

                case ',':
                    addWord(word);
                    word = ",";
                    addWord(word);
                    break;

                case ';':
                    addWord(word);
                    word = ";";
                    addWord(word);
                    break;

                case '-':
                    addWord(word);
                    word = "-";
                    addWord(word);
                    break;

                // Terminating strings
                case '.':
                    //Get the first word.
                    addWord(word);
                    //Now process .
                    word = ".";
                    addWord(word);
                    break;

                case '?':
                    addWord(word);
                    word = ".";
                    addWord(word);
                    break;

                case '!':
                    addWord(word);
                    word = ".";
                    addWord(word);
                    break;

                // Custom stuff
                case '\n':
                    addWord(word);
                    break;

                case '\'':
                    //Do nothing for now
                    break;
                case '"':
                    break;
                case '(':
                    break;
                case ')':
                    break;

                default:
//                    cout << c << endl;
                    word = word + c;
            }
        }
//        else if( word.empty() )
//        {
//            addWord(word);
//        }
    }

    file.close();

}


void CInputEnglish::addWord(string& word)
{
    if (word.empty()) {
        return;
    }

    //cout << word << endl;

    // Check beginning of text
    if (_markovChain.size() < _markovLength)
    {
        _markovChain.push_back(word);

        // Reached the correct size?
        if (_markovChain.size() == _markovLength)
            insertChain();
    }
    else
    {
        // Make the new chain
        _markovChain.pop_front();
        _markovChain.push_back(word);

        insertChain();
    }

    word.clear();
}


void CInputEnglish::insertChain()
{
    // Insert the chain
    CMap* previousMap = _map;

    for (auto x : _markovChain)
    {
        previousMap = &previousMap->insert(x);
    }
}


void CInputEnglish::setMarkovLength(const int &length)
{<commit_msg>comment<commit_after>//
//  CInputEnglish.cpp
//  Words
//
//  Created by <NAME> on 2014-03-03.
//  Copyright (c) 2014 Groarke & Co. All rights reserved.
//

#include "CInputEnglish.h"

CInputEnglish::CInputEnglish(CMap &map)
{
    _map = &map;
    _markovLength = 3;
}


void CInputEnglish::ReadStdin()
{

}


void CInputEnglish::ReadFile(string filename)
{

    ifstream file;
    file.open( filename.c_str(), ifstream::in );
    string word;
    word.clear();

    while( file.good() )
    {
        char c = file.get();   // get character from file

        if( file.good() )
        {
            switch( c )
            {
                case ' ':
//                    cout << word << endl;
                    addWord(word);
                    break;

                case ',':
                    addWord(word);
                    word = ",";
                    addWord(word);
                    break;

                case ';':
                    addWord(word);
                    word = ";";
                    addWord(word);
                    break;

                case '-':
                    addWord(word);
                    word = "-";
                    addWord(word);
                    break;

                // Terminating strings
                case '.':
                    //Get the first word.
                    addWord(word);
                    //Now process .
                    word = ".";
                    addWord(word);
                    break;

                case '?':
                    addWord(word);
                    word = ".";
                    addWord(word);
                    break;

                case '!':
                    addWord(word);
                    word = ".";
                    addWord(word);
                    break;

                // Custom stuff
                case '\n':
                    addWord(word);
                    break;

                case '\'':
                    //Do nothing for now
                    break;
                case '"':
                    break;
                case '(':
                    break;
                case ')':
                    break;

                default:
//                    cout << c << endl;
                    word = word + c;
            }
        }
//        else if( word.empty() )
//        {
//            addWord(word);
//        }
    }

    file.close();

}


void CInputEnglish::addWord(string& word)
{
    if (word.empty()) {
        return;
    }

    //cout << word << endl;

    // Check beginning of text
    if (_markovChain.size() < _markovLength)
    {
        _markovChain.push_back(word);

        // Reached the correct size?
        if (_markovChain.size() == _markovLength)
            insertChain();
    }
    else
    {
        // Make the new chain
        _markovChain.pop_front();
        _markovChain.push_back(word);

        insertChain();
    }

    word.clear();
}


void CInputEnglish::insertChain()
{
    // Point to our map, it will be changed after every insert.
    CMap* previousMap = _map;

    // Insert the chain
    for (auto x : _markovChain)
    {
        previousMap = &previousMap->insert(x);
    }
}


void CInputEnglish::setMarkovLength(const int &length)
{
{'fraction_non_alphanumeric': 0.1171754548257786, 'fraction_numerical': 0.004008633980881899, 'mean_word_length': 0.9260688836104513, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1499143', 'n_tokens_mistral': 1839, 'n_tokens_neox': 1660, 'n_words': 363}