<commit_before>/**
 * Created by Carson on 24/02/2017.
 * Implementation for the search function. Users enter their search terms in the search bar, and the key parts
 * are used to look through the database of posts/comments. Implements AutoTag algorithm from Algorithmia to
 * automatically get the key parts of the search terms. Note that numbers (e.g. course numbers) are not considered
 * to be words, and will need to be parsed out manually and searched for in the tags column.
 * Those words are then checked to see how relevant they
 * are to posts/comments in the database. This is done by Natural.
 *
 * need: npm install natural, npm install algorithmia
 *
 * AutoTag algorithm: https://algorithmia.com/algorithms/nlp/AutoTag
 * Natural Github: https://github.com/NaturalNode/natural
 *
 * Note: it is possible a small cost would be needed to use the AutoTag algorithm. Algorithmia credits are used to cover
 * the cost of the algorithm, which is estimated to be 11K credits per 10,000 calls. This would amount to ~1.17 USD per
 * 10,000 calls. However, for free, 5K credits are given each month, so on a small scale it is still free.
 */

var natural = require("natural");
var algorithmia = require("algorithmia");
var lit = require('./../Literals.js');
var log = require('./../log.js');
var dbr = require('./../DBRow.js');

var TfIdf = natural.TfIdf;
var wordRelater = new TfIdf();

//main function that ties it all together
function searchForContent(inputSearch) {
    getKeyTerms(inputSearch).then(function (keyTerms) {
        return searchForPosts(keyTerms);
    }).catch(function (error) {
        log.log("searchForContent error: " + error);
    });
}

//get the key parts of the search
function getKeyTerms(inputSearch) {
    return new Promise(function (resolve, reject) {
        algorithmia.client(lit.AUTO_TAG_API_KEY).algo(lit.AUTO_TAG_ALGORITHM).pipe(inputSearch)
            .then(function (response) {
                resolve(response.get());
            }, function (err) {
                reject(err);
            });
    });
}

//get the course numbers manually

//search through a post and get list of related ones, sorted as it's built
var arr = ["module"];
searchForPosts(arr);
function searchForPosts(keyTerms) {
    var documentInfo = [];
    var row = new dbr.DBRow(lit.POST_TABLE);
    return new Promise(function (resolve, reject) {
        row.query().then(function () {
            while (row.next()) {
                //search the post content and title
                var doc = row.getValue(lit.FIELD_TITLE) + "\n" + row.getValue(lit.FIELD_CONTENT);
                wordRelater.addDocument(doc);
                var docID = row.getValue(lit.FIELD_ID);
                var oneDoc = {measure: 0, id: docID};
                //add a row in the arrays for each document
                documentInfo.push(oneDoc);
            }
        }).then(function () {
            for (var termIndex in keyTerms) {
                wordRelater.tfidfs(keyTerms[termIndex], function (docIndex, measure) {
                    documentInfo[docIndex][lit.KEY_MEASURE] += measure;
                });
            }
            var out="";
            for (var i =0;i<documentInfo.length;i++){
                out+=documentInfo[i][lit.KEY_MEASURE]+", ";
            }
            log.log(out);
            documentInfo = removeLowRelations(documentInfo);
            resolve(sortDocumentsByRelation(documentInfo));
        }).catch(function (error) {
            log.log("searchForPosts error: " + error);
            reject(error);
        });
    });
}

function sortDocumentsByRelation(documentInfo) {
   /* var sortedPosts = [];
    var biggestMeasure = 0;
    var biggestIndex = -1;
    while (documentInfo.length > 0) {
        for (var i in documentInfo) {
            if (documentInfo[i][lit.KEY_MEASURE] > biggestMeasure) {
                biggestMeasure = documentInfo[i][lit.KEY_MEASURE];
                biggestIndex = i;
            }
        }
        sortedPosts.push(documentInfo[i][lit.FIELD_ID]);
        documentInfo.splice(i, 1);
    }
    log.log(sortedPosts);
    return sortedPosts;*/
   documentInfo=mergeSort(documentInfo);
   var output="";
   for (var i =0;i<documentInfo.length;i++){
       output+=documentInfo[i][lit.KEY_MEASURE]+", ";
   }
   log.log(output);
}

function removeLowRelations(documentInfo) {
    var i = 0;
    while (i < documentInfo.length) {
        if (documentInfo[i][lit.KEY_MEASURE] < lit.MIN_RELATION_MEASURE) { //remove the posts that aren't related enough
            documentInfo.splice(i, 1);
            //counter auto continued because the array decreased one size
        } else {
            i++;
        }
    }
    return documentInfo;<commit_msg>cleaned up previous commit (searcher.js mergesort)<commit_after>/**
 * Created by Carson on 24/02/2017.
 * Implementation for the search function. Users enter their search terms in the search bar, and the key parts
 * are used to look through the database of posts/comments. Implements AutoTag algorithm from Algorithmia to
 * automatically get the key parts of the search terms. Note that numbers (e.g. course numbers) are not considered
 * to be words, and will need to be parsed out manually and searched for in the tags column.
 * Those words are then checked to see how relevant they
 * are to posts/comments in the database. This is done by Natural.
 *
 * need: npm install natural, npm install algorithmia
 *
 * AutoTag algorithm: https://algorithmia.com/algorithms/nlp/AutoTag
 * Natural Github: https://github.com/NaturalNode/natural
 *
 * Note: it is possible a small cost would be needed to use the AutoTag algorithm. Algorithmia credits are used to cover
 * the cost of the algorithm, which is estimated to be 11K credits per 10,000 calls. This would amount to ~1.17 USD per
 * 10,000 calls. However, for free, 5K credits are given each month, so on a small scale it is still free.
 */

var natural = require("natural");
var algorithmia = require("algorithmia");
var lit = require('./../Literals.js');
var log = require('./../log.js');
var dbr = require('./../DBRow.js');

var TfIdf = natural.TfIdf;
var wordRelater = new TfIdf();

//main function that ties it all together
function searchForContent(inputSearch) {
    getKeyTerms(inputSearch).then(function (keyTerms) {
        return searchForPosts(keyTerms);
    }).catch(function (error) {
        log.log("searchForContent error: " + error);
    });
}

//get the key parts of the search
function getKeyTerms(inputSearch) {
    return new Promise(function (resolve, reject) {
        algorithmia.client(lit.AUTO_TAG_API_KEY).algo(lit.AUTO_TAG_ALGORITHM).pipe(inputSearch)
            .then(function (response) {
                resolve(response.get());
            }, function (err) {
                reject(err);
            });
    });
}

//get the course numbers manually

//search through a post and get list of related ones, sorted as it's built
function searchForPosts(keyTerms) {
    var documentInfo = [];
    var row = new dbr.DBRow(lit.POST_TABLE);
    return new Promise(function (resolve, reject) {
        row.query().then(function () {
            while (row.next()) {
                //search the post content and title
                var doc = row.getValue(lit.FIELD_TITLE) + "\n" + row.getValue(lit.FIELD_CONTENT);
                wordRelater.addDocument(doc);
                var docID = row.getValue(lit.FIELD_ID);
                var oneDoc = {measure: 0, id: docID};
                //add a row in the arrays for each document
                documentInfo.push(oneDoc);
            }
        }).then(function () {
            for (var termIndex in keyTerms) {
                wordRelater.tfidfs(keyTerms[termIndex], function (docIndex, measure) {
                    documentInfo[docIndex][lit.KEY_MEASURE] += measure;
                });
            }
            documentInfo = removeLowRelations(documentInfo);
            resolve(sortByMeasure(documentInfo));
        }).catch(function (error) {
            log.log("searchForPosts error: " + error);
            reject(error);
        });
    });
}

function removeLowRelations(documentInfo) {
    var i = 0;
    while (i < documentInfo.length) {
        if (documentInfo[i][lit.KEY_MEASURE] < lit.MIN_RELATION_MEASURE) { //remove the posts that aren't related enough
            documentInfo.splice(i, 1);
            //counter auto continued because the array decreased one size
        } else {
            i++;
        }
    }
    return documentInfo;
}

function sortByMeasure(documentInfo) {
    documentInfo = mergeSort(documentInfo);
    var sortedIDs = [];
    for (var index in documentInfo) {
        sortedIDs.push(documentInfo[index][lit.FIELD_ID]);
    }
    log.log(sortedIDs);
    return sortedIDs;
{'fraction_non_alphanumeric': 0.10822806418572892, 'fraction_numerical': 0.006828269033799932, 'mean_word_length': 2.5825519771708114, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2339964', 'n_tokens_mistral': 2537, 'n_tokens_neox': 2442, 'n_words': 777}