<commit_before>var request     = require('request')
    , mockery   = require('mockery')
    , sinon     = require('sinon')
    , should    = require('chai').should()
    , expect    = require('chai').expect
    , assert    = require('chai').assert
    , utils     = require('./../utils')
    , commons   = require('./../commons')
    , processor
    , cfg = {
    "maxConcurrentJobs": 10,
    "log" : true,
    "logger": {
        debug: function () {
        },
        info: function () {
        },
        warn: function () {
        },
        error: function () {
        }
    },
    "request_default_values": {
        "method": "GET",
        "timeout": 5,
        "ip": "unknown",
        "headers": {},
        "data": ""
    },
    "whiteList": ["*"]
    }

utils.configure(cfg);

describe('processor', function () {
    describe('processor functionality', function(){
        /**
         * In the processor utility we are using `request` module (https://github.com/request/request)
         * and in unit test we don't want to really make a request we use `mockery` && `sinon` to mock and stub
         * start preparation using:
         * require('mockery') && require('sinon')
         * mockery hijacks the require function and replaces modules with our mocks.
         * In the below code we register a sinon stub to be returned when require('request') is called.
         * Then we configure the mock in the test using the method .yield on the stub to a call the callback function passed to request with null/code for the error, an object for the response and another object for the body.
         * */
        var requestStub;

        before(function(){
            mockery.enable({
                warnOnReplace: false,
                warnOnUnregistered: false,
                useCleanCache: true
            });

            requestStub = sinon.stub();

            // replace the module `request` with a stub object
            mockery.registerMock('request', requestStub);
            processor = require('./../lib/processor');
            processor.configure(cfg);
        });

        after(function(){
            mockery.disable();
        });

        it('configure', function (done) {
            processor.configure.should.be.a('function');
            var configuration = processor.configure(cfg);
            configuration.should.be.a('object');
            done();
        });

        it('execute method - INVALID_TASK - Missing URL', function () {
            processor.run(
                [{
                    name: "INVALID_TASK",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 10000
                }], function (err, result) {
                    console.log(JSON.stringify(result));
                    result["INVALID_TASK"].body.should.equal(utils.builder.buildResponse(commons.CONST.RESPONSE_TYPE.INVALID_TASK).body);
                });
        });

        it('execute method - ETIMEDOUT', function () {
            requestStub.yields({code: "ETIMEDOUT"}, null, null);
            processor.run(
                [{
                    name: "ETIMEDOUT",
                    url: "http://notexisturl",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 100
                }], function (err, result) {
                    result["ETIMEDOUT"].body.should.equal(utils.builder.buildResponse(commons.CONST.RESPONSE_TYPE.ETIMEDOUT).body);
                });
        });

        it('run method - ERROR_API_URL', function () {
            requestStub.yields({error: "error"}, {response: "response"}, null);
            processor.run(
                [{
                    name: "ERROR_API_URL",
                    url: "htp://www.kojo.com",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 1000
                }], function (err, result) {
                    result["ERROR_API_URL"].body.should.equal(utils.builder.buildResponse(commons.CONST.RESPONSE_TYPE.ERROR_API_URL).body);
                });

        });

        it('run method - NO_JSON_OBJECT', function () {
            requestStub.yields(null, {statusCode: 200, headers: {bigHead: "bigHead"}}, "string");
            jobId = processor.run(
                [{
                    name: "NO_JSON_OBJECT",
                    url: "htp://www.kojo.com",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 1000
                }], function (err, result) {
                    result["NO_JSON_OBJECT"].body.should.equal(utils.builder.buildResponse(commons.CONST.RESPONSE_TYPE.NO_JSON_OBJECT).body);
                });

        });

        it('using param originalHeader', function () {
            before(function () {
                cfg.originalHeader = true;
                processor.configure(cfg);
            });
            requestStub.yields(null, {statusCode: 200, headers: {bigHead: "bigHead"}}, "string");
            jobId = processor.run(
                [{
                    name: "ORIGINAL_HEADERS",
                    url: "htp://www.kojo.com",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 1000
                }], function (err, result) {
                    result["ORIGINAL_HEADERS"].headers.originalHeader.should.be.define;
                });

        });

        it('using param originalHeader - false', function () {
            before(function () {
                cfg.originalHeader = false;
                processor.configure(cfg);
            });
            requestStub.yields(null, {statusCode: 200, headers: {bigHead: "bigHead"}}, "string");
            jobId = processor.run(
                [{
                    name: "ORIGINAL_HEADERS",
                    url: "htp://www.kojo.com",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 1000
                }], function (err, result) {
                    result["ORIGINAL_HEADERS"].headers.originalHeader.should.be.undefined;
                });
<commit_msg>fix test<commit_after>var request     = require('request')
    , mockery   = require('mockery')
    , sinon     = require('sinon')
    , should    = require('chai').should()
    , expect    = require('chai').expect
    , assert    = require('chai').assert
    , utils     = require('./../utils')
    , commons   = require('./../commons')
    , processor
    , cfg = {
    "maxConcurrentJobs": 10,
    "log" : true,
    "logger": {
        debug: function () {
        },
        info: function () {
        },
        warn: function () {
        },
        error: function () {
        }
    },
    "request_default_values": {
        "method": "GET",
        "timeout": 5,
        "ip": "unknown",
        "headers": {},
        "data": ""
    },
    "whiteList": ["*"]
    }

utils.configure(cfg);

describe('processor', function () {
    describe('processor functionality', function(){
        /**
         * In the processor utility we are using `request` module (https://github.com/request/request)
         * and in unit test we don't want to really make a request we use `mockery` && `sinon` to mock and stub
         * start preparation using:
         * require('mockery') && require('sinon')
         * mockery hijacks the require function and replaces modules with our mocks.
         * In the below code we register a sinon stub to be returned when require('request') is called.
         * Then we configure the mock in the test using the method .yield on the stub to a call the callback function passed to request with null/code for the error, an object for the response and another object for the body.
         * */
        var requestStub;

        before(function(){
            mockery.enable({
                warnOnReplace: false,
                warnOnUnregistered: false,
                useCleanCache: true
            });

            requestStub = sinon.stub();

            // replace the module `request` with a stub object
            mockery.registerMock('request', requestStub);
            processor = require('./../lib/processor');
            processor.configure(cfg);
        });

        after(function(){
            mockery.disable();
        });

        it('configure', function (done) {
            processor.configure.should.be.a('function');
            var configuration = processor.configure(cfg);
            configuration.should.be.a('object');
            done();
        });

        it('execute method - INVALID_TASK - Missing URL', function () {
            processor.run(
                [{
                    name: "INVALID_TASK",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 10000
                }], function (err, result) {
                    console.log(JSON.stringify(result));
                    result["INVALID_TASK"].body.should.equal(utils.builder.buildResponse(commons.CONST.RESPONSE_TYPE.INVALID_TASK).body);
                });
        });

        it('execute method - ETIMEDOUT', function () {
            requestStub.yields({code: "ETIMEDOUT"}, null, null);
            processor.run(
                [{
                    name: "ETIMEDOUT",
                    url: "http://notexisturl",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 100
                }], function (err, result) {
                    result["ETIMEDOUT"].body.should.equal(utils.builder.buildResponse(commons.CONST.RESPONSE_TYPE.ETIMEDOUT).body);
                });
        });

        it('run method - ERROR_API_URL', function () {
            requestStub.yields({error: "error"}, {response: "response"}, null);
            processor.run(
                [{
                    name: "ERROR_API_URL",
                    url: "htp://www.kojo.com",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 1000
                }], function (err, result) {
                    result["ERROR_API_URL"].body.should.equal(utils.builder.buildResponse(commons.CONST.RESPONSE_TYPE.ERROR_API_URL).body);
                });

        });

        it('run method - NO_JSON_OBJECT', function () {
            requestStub.yields(null, {statusCode: 200, headers: {bigHead: "bigHead"}}, "string");
            jobId = processor.run(
                [{
                    name: "NO_JSON_OBJECT",
                    url: "htp://www.kojo.com",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 1000
                }], function (err, result) {
                    result["NO_JSON_OBJECT"].body.should.equal(utils.builder.buildResponse(commons.CONST.RESPONSE_TYPE.NO_JSON_OBJECT).body);
                });

        });

        it('using param originalHeader', function () {
            before(function () {
                cfg.originalHeader = true;
                processor.configure(cfg);
            });
            requestStub.yields(null, {statusCode: 200, headers: {bigHead: "bigHead"}}, "string");
            jobId = processor.run(
                [{
                    name: "ORIGINAL_HEADERS",
                    url: "htp://www.kojo.com",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 1000
                }], function (err, result) {
                    result["ORIGINAL_HEADERS"].headers.originalHeader.should.be.a('string');
                });

        });

        it('using param originalHeader - false', function () {
            before(function () {
                cfg.originalHeader = false;
                processor.configure(cfg);
            });
            requestStub.yields(null, {statusCode: 200, headers: {bigHead: "bigHead"}}, "string");
            jobId = processor.run(
                [{
                    name: "ORIGINAL_HEADERS",
                    url: "htp://www.kojo.com",
                    encoding: "UTF8",
                    method: "GET",
                    retries: 3,
                    headers: {},
                    mimeType: "application/json",
                    data: "data",
                    timeout: 1000
                }], function (err, result) {
                    result["ORIGINAL_HEADERS"].headers.originalHeader.should.be.undefined;
                });

{'fraction_non_alphanumeric': 0.12517680339462517, 'fraction_numerical': 0.006789250353606789, 'mean_word_length': 1.342776673293572, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1865293', 'n_tokens_mistral': 3736, 'n_tokens_neox': 3439, 'n_words': 771}