<commit_before>#include <iostream>
#include <string>
#include <fstream> // files stuff
#include <vector>
#include <ctype.h>
#include <map> // multiple data structures are contained in maps, e.g. the dictionary, the TFIDF values, etc.
#include <ctime> // just for timing
#include "NaiveBayes.h"
#include "typedefs.h"
#include "doc.h"
#include "weight.h"
/*

INTRODUCTION ==============================================================
This code, written by <NAME>, is an implementation of the (term-weighted) 
Complement Naive Bayes Algorithm in C++. The paper that describes the algorithm,
and which was followed to write this code is:
< https://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf >


COMPILATION ===============================================================
TO COMPILE ON UBUNTU VIA COMMAND LINE: 
g++ main.cc NaiveBayes.cc typedefs.cc -lboost_filesystem -lboost_system -o main
or
g++ -L /usr/include/boost main.cc ...(rest is same as above)


HOW TO USE ================================================================
(Decide and make accordingly for most speed and flexibility)



*/
using namespace std;


int main()
{
    NaiveBayesClassifier clf;
    clock_t startTime = clock();
    cout << "Loading corpus into memory..." << endl;

    // INITIALIZATION
    // github link for sample corpii: <@TODO>
    clf.populateDocVec("Mini-Corpus"); // change this to where mini corpus is stored with cpp file (in project folder)

    string trainBool; // y if you want to train and create weights, anything else if you want to just read from file
    cout << "Would you like to train the algorithm, or pick up weights from the weights.txt file? y for yes (train from corpus - longer), anything else for no(train from weights.txt - quicker).";
    cout << endl;
    cin >> trainBool;
    if (trainBool == "y")
    {
        // calculate tfidfs, train weights and normalize them
        mapSDVec TFIDFvec = clf.prepAndFindTFIDFs(clf.docs, true);
        clf.naiveBayesTrain(TFIDFvec);
        clf.normalizeWeights();
    }
    else {
        clf.getWeightsFromFile();
    }

    cout << "Normalized weights retrieved" << endl;
    float secsElapsed1 = (float)(clock() - startTime)/CLOCKS_PER_SEC;

    cout << "Calculating accuracy..." << endl;

    ofstream resultsFile; // open file to print results to
    resultsFile.open ("results.txt", fstream::app); // this is just to clear file of old contents

    resultsFile << "Test Docs:" << endl;
    clf.checkAccuracy(clf.testDocs); // checks accuracy of test set
    resultsFile << "CV Docs:" << endl;
    clf.checkAccuracy(clf.cvDocs); // checks accuracy of cross validation set

    resultsFile.close();

    cout << endl << "===============================================" << endl;
    cout << "Time for assigning weights " << secsElapsed1 << endl;

    return 0;
}

<commit_msg>Change in instructions.<commit_after>#include <iostream>
#include <string>
#include <fstream> // files stuff
#include <vector>
#include <ctype.h>
#include <map> // multiple data structures are contained in maps, e.g. the dictionary, the TFIDF values, etc.
#include <ctime> // just for timing
#include "NaiveBayes.h"
#include "typedefs.h"
#include "doc.h"
#include "weight.h"


using namespace std;


int main()
{
    NaiveBayesClassifier clf;
    clock_t startTime = clock();
    cout << "Loading corpus into memory..." << endl;

    // INITIALIZATION
    // github link for sample corpii: <@TODO>
    clf.populateDocVec("Mini-Corpus"); // change this to where mini corpus is stored with cpp file (in project folder)

    string trainBool; // y if you want to train and create weights, anything else if you want to just read from file
    cout << "Would you like to train the algorithm, or pick up weights from the weights.txt file? y for yes (train from corpus - longer), anything else for no(train from weights.txt - quicker).";
    cout << endl;
    cin >> trainBool;
    if (trainBool == "y")
    {
        // calculate tfidfs, train weights and normalize them
        mapSDVec TFIDFvec = clf.prepAndFindTFIDFs(clf.docs, true);
        clf.naiveBayesTrain(TFIDFvec);
        clf.normalizeWeights();
    }
    else {
        clf.getWeightsFromFile();
    }

    cout << "Normalized weights retrieved" << endl;
    float secsElapsed1 = (float)(clock() - startTime)/CLOCKS_PER_SEC;

    cout << "Calculating accuracy..." << endl;

    ofstream resultsFile; // open file to print results to
    resultsFile.open ("results.txt", fstream::app); // this is just to clear file of old contents

    resultsFile << "Test Docs:" << endl;
    clf.checkAccuracy(clf.testDocs); // checks accuracy of test set
    resultsFile << "CV Docs:" << endl;
    clf.checkAccuracy(clf.cvDocs); // checks accuracy of cross validation set

    resultsFile.close();

    cout << endl << "===============================================" << endl;
    cout << "Time for assigning weights " << secsElapsed1 << endl;

    return 0;
}


{'fraction_non_alphanumeric': 0.16244849911712772, 'fraction_numerical': 0.0015695507161075143, 'mean_word_length': 3.101367658889783, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1702246', 'n_tokens_mistral': 1579, 'n_tokens_neox': 1467, 'n_words': 512}