<commit_before>var should = require('should');
var superagent = require('superagent');
var config = require('./config');
var chai = require('chai');
var expect = chai.expect;
var _ = require('underscore');
var async = require('async');
var server = require('./advancedServer.js');

var fs = require('fs');
var os = require('os');
var util = require('util');

var consumer = require('../../lib/consumer.js');
var listener = require('../../lib/listener.js');

//RUSH ENDPOINT
var HOST = config.rushServer.hostname;
var PORT = config.rushServer.port;
var RUSHENDPOINT = 'http://' + HOST + ':' + PORT;

//Final host endpoint
var fhHOST = config.simpleServerHostname;
var fhPORT = config.simpleServerPort;

var listenerLog = __dirname + '/Rush_listener_' + os.hostname() + '.log';
var consumerLog = __dirname + '/Rush_consumer_' + os.hostname() + '.log';

var RELAYREQUEST = "| lvl=INFO | op=RELAY REQUEST | msg=Relay Request received | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' |",
    PERSISTENCE_QUEUED = "| lvl=INFO | op=PERSISTENCE | msg=Persistence Completed | corr=N/A | trans=.* | hostname=.* | component=evPersistence | userID='.*' | state='queued'",
    PERSISTENCE_PROCESSING = "| lvl=INFO | op=PERSISTENCE | msg=Persistence Completed | corr=N/A | trans=.* | hostname=.* | component=evPersistence | userID='.*' | state='processing'",
    PERSISTENCE_COMPLETED = "| lvl=INFO | op=PERSISTENCE | msg=Persistence Completed | corr=N/A | trans=.* | hostname=.* | component=evPersistence | userID='.*' | state='completed'",
    PERSISTENCE_ERROR = "| lvl=INFO | op=PERSISTENCE | msg=Persistence Completed | corr=N/A | trans=.* | hostname=.* | component=evPersistence | userID='.*' | state='error'",
    JOBENDED = "| lvl=INFO | op=CONSUME | msg=Job Ended | corr=N/A | trans=.* | hostname=.* | component=consumer | userID='.*' |",
    NOHOST = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter',  parameter: 'x-relayer-host',  userMessage: 'Valid format: host[:port]' }]",
    INVALIDPERSISTENCE = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter_accepted_values',  parameter: 'x-relayer-persistence',  acceptedValues: [ 'BODY', 'STATUS', 'HEADER' ] }]",
    INVALID_CALLBACK_PROTO = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter',  parameter: 'x-relayer-httpcallback',  userMessage: 'Protocol is not defined' }]",
    INVALID_CALLBACK_HOST = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter',  parameter: 'x-relayer-httpcallback',  userMessage: 'Hostname expected. Empty host after protocol' }]"
    INVALID_RETRY = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter',  parameter: 'x-relayer-retry',  userMessage: 'Invalid retry value: .*' }]",
    INVALID_HEADER = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter',  parameter: 'x-relayer-header',  userMessage: 'Value for header .* is not defined' }]",
    JOBERROR = "| lvl=WARNING | op=DO JOB | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=eventWorker | userID='.*' | error={ [Error: getaddrinfo ENOTFOUND]  code: 'ENOTFOUND',  errno: 'ENOTFOUND',  syscall: 'getaddrinfo',  resultOk: false }",
    CALLBACKERROR = "| lvl=WARNING | op=HTTP CALLBACK | msg=Callback Error | corr=N/A | trans=.* | hostname=.* | component=evCallback | userID='.*' | error={ callback_err: 'getaddrinfo ENOTFOUND' }"

var escape = function(text) {
  return text.replace(/[-[\]{}()+?,\\^$|#\s]/g, "\\$&");
};

ENDPOINT = fhHOST + ':' + fhPORT;

var serversToShutDown = [];

// Time to wait to check the status of the task
var TIMEOUT = 1000;
var CREATED = 201;
var describeTimeout = 5000;
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; //Accept self signed certs

function _scenario(data){

  it(data.name +  ' #LOGS', function(done){
    var agent = superagent.agent();
    var id;

    var method;
      switch(data.method){
        case "DELETE":
          method = 'del';
          break;
        default:
          method = data.method.toLowerCase()
      }
    var simpleServer = server({port : fhPORT, protocol : data.protocol}, {},
      function() {

        data.expected.push(RELAYREQUEST);
        var req = agent
            [method](RUSHENDPOINT + data.path )
            .set('x-relayer-host', ENDPOINT)  //Always the same endpoint
            .set('x-relayer-persistence','BODY')
            .set('content-type','application/json')
            .set(data.headers)
            if(data.method === 'POST' || data.method === 'PUT'){
              req = req.send(data.body);
            }
            req.end(function(){

              setTimeout(function() {

                var lLog = fs.readFileSync(listenerLog).toString();
                var cLog = fs.readFileSync(consumerLog).toString();

                console.log('\033[31m' + lLog);

                for(var i=0; i < data.expected.length; i++){
                  var pattern=new RegExp(escape(data.expected[i]));
                  var contains = pattern.test(lLog);
                  contains.should.be.true;
                }
                done();

              }, TIMEOUT);
            });
      },
      function(dataReceived) {});
    serversToShutDown.push(simpleServer);
  });
}
<commit_msg>coloring logs (travis testing purposes only)<commit_after>var should = require('should');
var superagent = require('superagent');
var config = require('./config');
var chai = require('chai');
var expect = chai.expect;
var _ = require('underscore');
var async = require('async');
var server = require('./advancedServer.js');

var fs = require('fs');
var os = require('os');
var util = require('util');

var consumer = require('../../lib/consumer.js');
var listener = require('../../lib/listener.js');

//RUSH ENDPOINT
var HOST = config.rushServer.hostname;
var PORT = config.rushServer.port;
var RUSHENDPOINT = 'http://' + HOST + ':' + PORT;

//Final host endpoint
var fhHOST = config.simpleServerHostname;
var fhPORT = config.simpleServerPort;

var listenerLog = __dirname + '/Rush_listener_' + os.hostname() + '.log';
var consumerLog = __dirname + '/Rush_consumer_' + os.hostname() + '.log';

var RELAYREQUEST = "| lvl=INFO | op=RELAY REQUEST | msg=Relay Request received | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' |",
    PERSISTENCE_QUEUED = "| lvl=INFO | op=PERSISTENCE | msg=Persistence Completed | corr=N/A | trans=.* | hostname=.* | component=evPersistence | userID='.*' | state='queued'",
    PERSISTENCE_PROCESSING = "| lvl=INFO | op=PERSISTENCE | msg=Persistence Completed | corr=N/A | trans=.* | hostname=.* | component=evPersistence | userID='.*' | state='processing'",
    PERSISTENCE_COMPLETED = "| lvl=INFO | op=PERSISTENCE | msg=Persistence Completed | corr=N/A | trans=.* | hostname=.* | component=evPersistence | userID='.*' | state='completed'",
    PERSISTENCE_ERROR = "| lvl=INFO | op=PERSISTENCE | msg=Persistence Completed | corr=N/A | trans=.* | hostname=.* | component=evPersistence | userID='.*' | state='error'",
    JOBENDED = "| lvl=INFO | op=CONSUME | msg=Job Ended | corr=N/A | trans=.* | hostname=.* | component=consumer | userID='.*' |",
    NOHOST = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter',  parameter: 'x-relayer-host',  userMessage: 'Valid format: host[:port]' }]",
    INVALIDPERSISTENCE = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter_accepted_values',  parameter: 'x-relayer-persistence',  acceptedValues: [ 'BODY', 'STATUS', 'HEADER' ] }]",
    INVALID_CALLBACK_PROTO = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter',  parameter: 'x-relayer-httpcallback',  userMessage: 'Protocol is not defined' }]",
    INVALID_CALLBACK_HOST = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter',  parameter: 'x-relayer-httpcallback',  userMessage: 'Hostname expected. Empty host after protocol' }]"
    INVALID_RETRY = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter',  parameter: 'x-relayer-retry',  userMessage: 'Invalid retry value: .*' }]",
    INVALID_HEADER = "| lvl=WARNING | op=ASSIGN REQUEST | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=listener | userID='.*' | error=[{ type: 'invalid_parameter',  parameter: 'x-relayer-header',  userMessage: 'Value for header .* is not defined' }]",
    JOBERROR = "| lvl=WARNING | op=DO JOB | msg=Request Error | corr=N/A | trans=.* | hostname=.* | component=eventWorker | userID='.*' | error={ [Error: getaddrinfo ENOTFOUND]  code: 'ENOTFOUND',  errno: 'ENOTFOUND',  syscall: 'getaddrinfo',  resultOk: false }",
    CALLBACKERROR = "| lvl=WARNING | op=HTTP CALLBACK | msg=Callback Error | corr=N/A | trans=.* | hostname=.* | component=evCallback | userID='.*' | error={ callback_err: 'getaddrinfo ENOTFOUND' }"

var escape = function(text) {
  return text.replace(/[-[\]{}()+?,\\^$|#\s]/g, "\\$&");
};

ENDPOINT = fhHOST + ':' + fhPORT;

var serversToShutDown = [];

// Time to wait to check the status of the task
var TIMEOUT = 1000;
var CREATED = 201;
var describeTimeout = 5000;
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; //Accept self signed certs

function _scenario(data){

  it(data.name +  ' #LOGS', function(done){
    var agent = superagent.agent();
    var id;

    var method;
      switch(data.method){
        case "DELETE":
          method = 'del';
          break;
        default:
          method = data.method.toLowerCase()
      }
    var simpleServer = server({port : fhPORT, protocol : data.protocol}, {},
      function() {

        data.expected.push(RELAYREQUEST);
        var req = agent
            [method](RUSHENDPOINT + data.path )
            .set('x-relayer-host', ENDPOINT)  //Always the same endpoint
            .set('x-relayer-persistence','BODY')
            .set('content-type','application/json')
            .set(data.headers)
            if(data.method === 'POST' || data.method === 'PUT'){
              req = req.send(data.body);
            }
            req.end(function(){

              setTimeout(function() {

                var lLog = fs.readFileSync(listenerLog).toString();
                var cLog = fs.readFileSync(consumerLog).toString();

                console.log('+++++++++++++++++++++++++++++++++++++++++++++++++');
                console.log('\033[31m' + lLog);
                console.log('+++++++++++++++++++++++++++++++++++++++++++++++++');

                for(var i=0; i < data.expected.length; i++){
                  var pattern=new RegExp(escape(data.expected[i]));
                  var contains = pattern.test(lLog);
                  contains.should.be.true;
                }
                done();

              }, TIMEOUT);
            });
      },
      function(dataReceived) {});
    serversToShutDown.push(simpleServer);
  });
}

{'fraction_non_alphanumeric': 0.18141928951143363, 'fraction_numerical': 0.0030377183360054006, 'mean_word_length': 3.4223880597014924, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7517626', 'n_tokens_mistral': 3912, 'n_tokens_neox': 3799, 'n_words': 835}