<commit_before>"use strict";

const request = require('request');
const Promise = require('promise');
const entities = require("html-entities");

const htmlentities = new entities.AllHtmlEntities();

class QnAContext {
    constructor(qnaMakerKey, id, name, docId, kbid, score, similarContexts, possibleQuestions) {
        this.id = id;
        this.name = name;
        this.docId = docId;
        this.kbid = kbid;
        this._qnaMakerKey = qnaMakerKey;
        this.score = score;
        this.similarContexts = similarContexts;
        this.possibleQuestions = possibleQuestions;
    }

    static fromState(contextState) {
        return new QnAContext(contextState._qnaMakerKey, contextState.id, contextState.name, contextState.docId, contextState.kbid, contextState.score, contextState.similarContexts, contextState.possibleQuestions);
    }

    scoreQuestion(question) {
        return new Promise((resolve, reject) => {
            let numberOfRespones = 3;
            let _this = this;
            var postBody = '{"question":"' + question + '", "top":' + numberOfRespones + '}';
            request({
                url: 'https://westus.api.cognitive.microsoft.com/qnamaker/v2.0/knowledgebases/' + this.kbid + '/generateanswer',
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                    'Ocp-Apim-Subscription-Key': this._qnaMakerKey
                },
                body: postBody
            }, (error, response, body) => {
                var result;
                let isSuccessful = !error && response.statusCode === 200;
                try {
                    if (isSuccessful) {
                        result = JSON.parse(body);
                        var answerEntities = [];
                        if (result.answers !== null && result.answers.length > 0) {
                            result.answers.forEach(function (ans) {
                                ans.score /= 100;
                                ans.answer = htmlentities.decode(ans.answer);
                                var answerEntity = {
                                    context: _this,
                                    questionMatched: ans.questions[0],
                                    score: ans.score,
                                    entity: ans.answer.replace(/\n\n/g, '\n\n &nbsp; \n\n').replace(/\.\n/g, '.\n\n').replace(/\./g, '. ').replace(/\n\nPage \d of \d\n\n/g, '').replace(/[a-z]\n[A-Z]/g, function (match) {
                                        return match.replace('\n', '\n\n'); // Separate out subtitles
                                      }).replace(/[a-z]\n[a-z]/g, function (match) {  
                                          return match.replace('\n', ' '); // Lose random newlines from document wrapping
                                      }),
                                    type: 'answer',
                                    kbid: _this.kbid,
                                    name: _this.name,
                                    docId: _this.docId
                                };<commit_msg>Added explanation of regex replacements for answers<commit_after>"use strict";

const request = require('request');
const Promise = require('promise');
const entities = require("html-entities");

const htmlentities = new entities.AllHtmlEntities();

class QnAContext {
    constructor(qnaMakerKey, id, name, docId, kbid, score, similarContexts, possibleQuestions) {
        this.id = id;
        this.name = name;
        this.docId = docId;
        this.kbid = kbid;
        this._qnaMakerKey = qnaMakerKey;
        this.score = score;
        this.similarContexts = similarContexts;
        this.possibleQuestions = possibleQuestions;
    }

    static fromState(contextState) {
        return new QnAContext(contextState._qnaMakerKey, contextState.id, contextState.name, contextState.docId, contextState.kbid, contextState.score, contextState.similarContexts, contextState.possibleQuestions);
    }

    scoreQuestion(question) {
        return new Promise((resolve, reject) => {
            let numberOfRespones = 3;
            let _this = this;
            var postBody = '{"question":"' + question + '", "top":' + numberOfRespones + '}';
            request({
                url: 'https://westus.api.cognitive.microsoft.com/qnamaker/v2.0/knowledgebases/' + this.kbid + '/generateanswer',
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                    'Ocp-Apim-Subscription-Key': this._qnaMakerKey
                },
                body: postBody
            }, (error, response, body) => {
                var result;
                let isSuccessful = !error && response.statusCode === 200;
                try {
                    if (isSuccessful) {
                        result = JSON.parse(body);
                        var answerEntities = [];
                        if (result.answers !== null && result.answers.length > 0) {
                            result.answers.forEach(function (ans) {
                                ans.score /= 100;
                                ans.answer = htmlentities.decode(ans.answer);
                                var answerEntity = {
                                    context: _this,
                                    questionMatched: ans.questions[0],
                                    score: ans.score,
                                    // The below replacements are temporary - ideally these issues should be fixed during preprocessing (other than the first one which is bot-specific)
                                    entity: ans.answer.replace(/\n\n/g, '\n\n &nbsp; \n\n').replace(/\.\n/g, '.\n\n').replace(/\./g, '. ').replace(/\n\nPage \d of \d\n\n/g, '').replace(/[a-z]\n[A-Z]/g, function (match) {
                                        return match.replace('\n', '\n\n'); // Separate out subtitles
                                      }).replace(/[a-z]\n[a-z]/g, function (match) {  
                                          return match.replace('\n', ' '); // Lose random newlines from document wrapping
                                      }),
                                    type: 'answer',
                                    kbid: _this.kbid,
                                    name: _this.name,
                                    docId: _this.docId
                                };
{'fraction_non_alphanumeric': 0.11750192752505782, 'fraction_numerical': 0.003392444101773323, 'mean_word_length': 1.3280689160086145, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 29, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5252761', 'n_tokens_mistral': 1722, 'n_tokens_neox': 1621, 'n_words': 343}