<commit_before>var request = require('request').defaults({
    'proxy': process.env.http_proxy || process.env.HTTP_PROXY,
    'timeout': 20 * 1000
});
var querystring = require('querystring');
var xml2js = require('xml2js');

var config = require('./config.json');

var supported_langs = ['en', 'hi', 'ur'];

function getAccessToken() {
    return new Promise(function (resolve, reject) {
        var options = {
            url: 'https://api.cognitive.microsoft.com/sts/v1.0/issueToken',
            headers: {
                'Ocp-Apim-Subscription-Key': config.microsoftTranslateSubscriptionKey
            },
        };
        request.post(options, function (error, response, body) {
            if (error) reject(error);
            else resolve(body);
        });
    });
}

function translate (text, from, to) {
    if (supported_langs.indexOf(from) < 0 || supported_langs.indexOf(to) < 0) {
        return Promise.reject("LANG_NOT_SUPPORTED");
    }

    return getAccessToken().then(function (token) {
        var url = 'http://api.microsofttranslator.com/v2/Http.svc/Translate';
        var data = {
            appid: 'Bearer ' + token,
            from: from,
            to: to,
            text: text,
        };

        url = url + '?' + querystring.stringify(data);

        return new Promise(function(resolve, reject) {
            request(url, function (err, res, body) {
                if (err || res.statusCode !== 200) {
                    reject(err);
                } else {
                    xml2js.parseString(body, function (err, result) {
                        if (err) {
                            reject(err);
                        } else {
                            var targetText = result.string._;
                            var sourceSentences = text.split("\n");
                            var targetSentences = targetText.split("\n");
                            var sentences = [];

                            if (sourceSentences.length != targetSentences.length) {
                                // FIXME: find a better way to handle this
                                sentences.push({
                                    source: text,
                                    target: targetText
                                });
                            } else {
                                for (var i = 0; i < sourceSentences.length; i++) {
                                    sentences.push({
                                        source: sourceSentences[i].trim(),
                                        target: targetSentences[i].trim()
                                    });
                                }
                            }

                            resolve({
                                'text': targetText,
                                'sentences': sentences
                            });
                        }
                    });
                }
            });
        });
    });
}

module.exports = translate;<commit_msg>Improve Microsoft Translation API integration<commit_after>var request = require('request').defaults({
    'proxy': process.env.http_proxy || process.env.HTTP_PROXY,
    'timeout': 20 * 1000
});
var querystring = require('querystring');
var xml2js = require('xml2js');

var config = require('./config.json');

var supported_langs = ['en', 'hi', 'ur'];

function getAccessToken() {
    return new Promise(function (resolve, reject) {
        var options = {
            url: 'https://api.cognitive.microsoft.com/sts/v1.0/issueToken',
            headers: {
                'Ocp-Apim-Subscription-Key': config.microsoftTranslateSubscriptionKey
            },
        };
        request.post(options, function (error, response, body) {
            if (error) reject(error);
            else resolve(body);
        });
    });
}

function translate (text, from, to) {
    if (supported_langs.indexOf(from) < 0 || supported_langs.indexOf(to) < 0) {
        return Promise.reject("LANG_NOT_SUPPORTED");
    }

    var sourceSentences = text.split("\n").filter(function (sentence) {
        return sentence;
    });

    return getAccessToken().then(function (token) {
        var url = 'http://api.microsofttranslator.com/v2/Http.svc/TranslateArray';

        var data = {
            AppId: 'Bearer ' + token,
            From: from,
            Texts: {
                string: sourceSentences.map(function (sentence) {
                    return {
                        $: {
                            xmlns: "http://schemas.microsoft.com/2003/10/Serialization/Arrays"
                        },
                        _: sentence
                    };
                })
            },
            To: to,
        };

        var xmlBuilder = new xml2js.Builder({
            rootName: "TranslateArrayRequest",
            headless: true
        });
        var requestXml = xmlBuilder.buildObject(data);

        return new Promise(function(resolve, reject) {
            request.post(
                {
                    url: url,
                    body: requestXml,
                    headers: {
                        "Content-Type": "application/xml"
                    }
                },
                function (err, res, body) {
                    if (err || res.statusCode !== 200) {
                        reject(err);
                    } else {
                        xml2js.parseString(body, function (err, result) {
                            if (err) {
                                reject(err);
                            } else {
                                var targetSentences = result.ArrayOfTranslateArrayResponse.TranslateArrayResponse
                                    .map(function (trans) {
                                        return trans.TranslatedText[0];
                                    });
                                var sentences = [];

                                if (sourceSentences.length != targetSentences.length) {
                                    reject("MS_TRANSLATION_ERROR");
                                } else {
                                    for (var i = 0; i < sourceSentences.length; i++) {
                                        sentences.push({
                                            source: sourceSentences[i].trim(),
                                            target: targetSentences[i].trim()
                                        });
                                    }
                                }

                                resolve({
                                    'text': sentences.map(function (sentence) {
                                        return sentence.target;
                                    }).join("\n"),
                                    'sentences': sentences
                                });
                            }
                        });
                    }
                }
            );
        });
    });
}

module.exports = translate;
{'fraction_non_alphanumeric': 0.10635280771412366, 'fraction_numerical': 0.006239364719228588, 'mean_word_length': 1.024978466838932, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7035897', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1572, 'n_words': 318}