<commit_before>/**
 * converter.h
 * Copyright (C) 2015 <NAME> <<EMAIL>>
 *
 * Distributed under terms of the MIT license.
 */

#ifndef CONVERTER_H
#define CONVERTER_H
#include <string>
#include <list>
#include <QtSql>
#include <sstream>

#include "../Tokenizer/stokenize.h"
#include "../../CONFIG/config.h"
#include "word.h"

using namespace std;

namespace NLP
{

    /// Relative path of where Dictionary database is
    /// Note : Path is relative to where "build" folder is generated
    const QString DB_PATH = "../../en_db.sqlite";

    /**
     * @brief Class to process a sentence and generate the list of words and it's corresponding datas
     */
    class Converter
    {
        private:
            string      mSentence;          // original sentence
            list<Token> mTokens;            // store all essential tokens to be processed

            static QSqlDatabase mDb;

            // Helper function
            void          extractTokens();
            set<WordType> getWordTypes(string wordname, size_t recurDepth = 0);

        public:
            Converter(const string& sentence);

            list<Word>  getWords();
            ~Converter();
    };

    QSqlDatabase Converter::mDb = QSqlDatabase::addDatabase("QSQLITE");

    /**
     * @brief Constructor first setup the list of tokens from a sentence
     *        Then, connect Dictionary database to a private QSqlDatabase variable
     *        This database will be used by any query created in this class
     * @param sentence
     */
    Converter::Converter(const string &sentence) :
        mSentence(sentence)
    {
        extractTokens(); // extract tokens
        mDb.setDatabaseName(DB_PATH); // database in main project
        if( !mDb.open() ) {
            qDebug() << mDb.lastError();
            qFatal( "Failed to connect to database." );
            throw std::invalid_argument("Error: Invalid database");
        } else { qDebug() << "Debug: Database opened." << endl; }
    }


    /**
     * @brief Function only extract alphabet for now, future implementation might also include comma, question mark
     *     BUG : Inside Tokenizer, remove all occurences of '
     *           Ex: can't -> cant, or
     *           Transform the sentences so that special kinda of words gets transformed
     *           Regex ?
     *           Add extra layer of string processsing before Tokenizing
     */
    void Converter::extractTokens()
    {
        STokenize mySTokenize(mSentence);
        while (mySTokenize.More()) {
            mTokens.push_back(mySTokenize.nextToken());
        }
    }

    /**
     * @brief Function takes a 'word' and return a set of possible WordTypes
     *        * Helper function to wrap the request database
     *        QSqlDat, QSqlQuery has to be defined on same local scope for it to work
     * @note  Has recursive algorithm
     * @param word in string
     * @return set of WordType (or tags)
     */
    set<WordType> Converter::getWordTypes(string wordname, size_t recurDepth)
    {

        /// Start query process
        QSqlQuery       mLiteQr;
        string          qrStr = "SELECT DISTINCT wordtype FROM entries WHERE word = '" + wordname + "'";
        string          rawTypesCollections;
        set<string>     uniqueWT; // Create vector to hold our words

        /// Run, then Check if the query is valid
        mLiteQr.prepare(qrStr.c_str());
        if( !mLiteQr.exec() ) {
            qDebug() << mLiteQr.lastError();
            throw std::invalid_argument("Invalid query.");
         }

        /// Extracting process
        QSqlRecord rec = mLiteQr.record();
        while( mLiteQr.next() ) {
            rawTypesCollections += string(" "); /// Add spaces in-between to make extracting easier
            rawTypesCollections += mLiteQr.value(0).toString().toStdString();
        }

        /// Recursive Method
        /// If no queries is found, the word might be in plural form, use recursive method to extract lesser string
        /// NOTE : This is not the optimal method
        /// check if wordtype query is empty, word is not too short, and not cut more than 2 times
        if (rawTypesCollections.empty() && wordname.length() > 1 && recurDepth < 3) {
//            cout << "len WT of " << wordname << " : " << rawTypesCollections.length() << endl;
//            cout << " calling recursive for : " << wordname << endl;
            return getWordTypes(wordname.substr(0, wordname.length() - 1), recurDepth + 1);
        }

        /// Clean up comma, and maybe other weird character in wordtypes if necessary
        rawTypesCollections.erase(std::remove(rawTypesCollections.begin(),
                                              rawTypesCollections.end(), ',')
                                  , rawTypesCollections.end());
        rawTypesCollections.erase(std::remove(rawTypesCollections.begin(),
                                              rawTypesCollections.end(), '&')
                                  , rawTypesCollections.end());
        string       buf; // Have a buffer string
        stringstream ss(rawTypesCollections); // Insert the string into a stream

        while (ss >> buf)
            uniqueWT.insert(buf);

        /// Converting sets of string to sets of WordTypes
        set<WordType>   WordTypes;
        for(string WT : uniqueWT)
            WordTypes.insert(WordTypeMap[WT]);
        return WordTypes;
    }

    /**
     * @brief Function to finalize the list of words to be returned, along with its corresponding roles
     * @return list<Word>
     */
    list<Word> Converter::getWords()
    {
        // NOTE : Move database declaration here

        list<Word>          WordList;
        set <WordType>      foundTypes;<commit_msg>changed list<word> to vector<word><commit_after>/**
 * converter.h
 * Copyright (C) 2015 <NAME> <<EMAIL>>
 *
 * Distributed under terms of the MIT license.
 */

#ifndef CONVERTER_H
#define CONVERTER_H
#include <string>
#include <list>
#include <QtSql>
#include <sstream>

#include "../Tokenizer/stokenize.h"
#include "../../CONFIG/config.h"
#include "word.h"

using namespace std;

namespace NLP
{

    /// Relative path of where Dictionary database is
    /// Note : Path is relative to where "build" folder is generated
    const QString DB_PATH = "../../en_db.sqlite";

    /**
     * @brief Class to process a sentence and generate the list of words and it's corresponding datas
     */
    class Converter
    {
        private:
            string      mSentence;          // original sentence
            list<Token> mTokens;            // store all essential tokens to be processed

            static QSqlDatabase mDb;

            // Helper function
            void          extractTokens();
            set<WordType> getWordTypes(string wordname, size_t recurDepth = 0);

        public:
            Converter(const string& sentence);

            vector<Word>  getWords();
            ~Converter();
    };

    QSqlDatabase Converter::mDb = QSqlDatabase::addDatabase("QSQLITE");

    /**
     * @brief Constructor first setup the list of tokens from a sentence
     *        Then, connect Dictionary database to a private QSqlDatabase variable
     *        This database will be used by any query created in this class
     * @param sentence
     */
    Converter::Converter(const string &sentence) :
        mSentence(sentence)
    {
        extractTokens(); // extract tokens
        mDb.setDatabaseName(DB_PATH); // database in main project
        if( !mDb.open() ) {
            qDebug() << mDb.lastError();
            qFatal( "Failed to connect to database." );
            throw std::invalid_argument("Error: Invalid database");
        } else { qDebug() << "Debug: Database opened." << endl; }
    }


    /**
     * @brief Function only extract alphabet for now, future implementation might also include comma, question mark
     *     BUG : Inside Tokenizer, remove all occurences of '
     *           Ex: can't -> cant, or
     *           Transform the sentences so that special kinda of words gets transformed
     *           Regex ?
     *           Add extra layer of string processsing before Tokenizing
     */
    void Converter::extractTokens()
    {
        STokenize mySTokenize(mSentence);
        while (mySTokenize.More()) {
            mTokens.push_back(mySTokenize.nextToken());
        }
    }

    /**
     * @brief Function takes a 'word' and return a set of possible WordTypes
     *        * Helper function to wrap the request database
     *        QSqlDat, QSqlQuery has to be defined on same local scope for it to work
     * @note  Has recursive algorithm
     * @param word in string
     * @return set of WordType (or tags)
     */
    set<WordType> Converter::getWordTypes(string wordname, size_t recurDepth)
    {

        /// Start query process
        QSqlQuery       mLiteQr;
        string          qrStr = "SELECT DISTINCT wordtype FROM entries WHERE word = '" + wordname + "'";
        string          rawTypesCollections;
        set<string>     uniqueWT; // Create vector to hold our words

        /// Run, then Check if the query is valid
        mLiteQr.prepare(qrStr.c_str());
        if( !mLiteQr.exec() ) {
            qDebug() << mLiteQr.lastError();
            throw std::invalid_argument("Invalid query.");
         }

        /// Extracting process
        QSqlRecord rec = mLiteQr.record();
        while( mLiteQr.next() ) {
            rawTypesCollections += string(" "); /// Add spaces in-between to make extracting easier
            rawTypesCollections += mLiteQr.value(0).toString().toStdString();
        }

        /// Recursive Method
        /// If no queries is found, the word might be in plural form, use recursive method to extract lesser string
        /// NOTE : This is not the optimal method
        /// check if wordtype query is empty, word is not too short, and not cut more than 2 times
        if (rawTypesCollections.empty() && wordname.length() > 1 && recurDepth < 3) {
//            cout << "len WT of " << wordname << " : " << rawTypesCollections.length() << endl;
//            cout << " calling recursive for : " << wordname << endl;
            return getWordTypes(wordname.substr(0, wordname.length() - 1), recurDepth + 1);
        }

        /// Clean up comma, and maybe other weird character in wordtypes if necessary
        rawTypesCollections.erase(std::remove(rawTypesCollections.begin(),
                                              rawTypesCollections.end(), ',')
                                  , rawTypesCollections.end());
        rawTypesCollections.erase(std::remove(rawTypesCollections.begin(),
                                              rawTypesCollections.end(), '&')
                                  , rawTypesCollections.end());
        string       buf; // Have a buffer string
        stringstream ss(rawTypesCollections); // Insert the string into a stream

        while (ss >> buf)
            uniqueWT.insert(buf);

        /// Converting sets of string to sets of WordTypes
        set<WordType>   WordTypes;
        for(string WT : uniqueWT)
            WordTypes.insert(WordTypeMap[WT]);
        return WordTypes;
    }

    /**
     * @brief Function to finalize the list of words to be returned, along with its corresponding roles
     * @return list<Word>
     */
    vector<Word> Converter::getWords()
    {
        // NOTE : Move database declaration here

        vector<Word>          WordList;
        set <WordType>      foundTypes;
{'fraction_non_alphanumeric': 0.09966201577259728, 'fraction_numerical': 0.0020799029378628995, 'mean_word_length': 2.0052083333333335, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2531244', 'n_tokens_mistral': 3025, 'n_tokens_neox': 2917, 'n_words': 994}