<commit_before>'use strict';
/* eslint-env es6 */
/* eslint-disable no-unused-vars */

// jscs:disable disallowMultipleVarDecl
var Q = require('q');
var Hoxy = require('hoxy');
var Mongoose = require('mongoose');
Mongoose.Promise = require('q').Promise;
var mongoDB;
var Prettyjson = require('prettyjson');
var _ = require('lodash');
var __ = require('./lodash-like.js');

exports.serve = function(configParams) {
  // jshint validthis:true
  var servicesShemaModel =
    {
      url: String,
      method: String,
      headers: Object,
      body: Object,
      responseData: Object,
      responseStatusCode: Number,
      timeStamp: Date
    },
    ServicesSchema, proxy;

  function formatHeaders(headers) {
    return _.omit(headers, ['host', 'user-agent', 'accept', 'accept-language', 'accept-encoding', 'dnt',
      'x-requested-with', 'content-type', 'referer', 'content-length', 'cookie', 'origin',
      'connection', 'pragma', 'cache-control']);
  }

  function respondEntry(response, entry) {
    var responseData = entry.responseData || entry.successData; // <- legacy support.
    response.statusCode = entry.responseStatusCode || 200;
    response.string = JSON.stringify(responseData);
  }

  function updateRequestEntry(request, response, entry) {
    entry.responseData = response.json;
    return entry;
  }

  function createRequestEntry(request, response) {
    (new ServicesSchema({
      url: request.url,
      method: request.method,
      headers: __.sortObjectDeep(formatHeaders(request.headers), true),
      body: __.sortObjectDeep(request.json, true),
      responseData: response.json,
      responseStatusCode: response.statusCode,
      timeStamp: Date.now()
    })).save(function(error) {
      if (error) return console.error(error);
    });
  }

  function responseInterceptor(request, response) {
    createRequestEntry(request, response);
    if (!configParams.quiet) {
      requestPrintLog(request);
    }
  }

  function printSearchNeedle(searchNeedle) {
    console.info('===================== Serach Needle =======================');
    console.info(Prettyjson.render({searchedTimeStamp: String(Date.now())}));
    console.info();
    console.info(Prettyjson.render(__.sortObjectDeep(searchNeedle.source, true)));
  }

  function requestPrintLog(request) {
    console.info('============ Request from greenTomato -> server ===============');
    console.info(Prettyjson.render({requestTimeStamp: String(Date.now())}));
    console.info();
    console.info(Prettyjson.render(request));
  }

  function getSearchNeedle(request) {
    var searchNeedle =
      {
        url: _.clone(request.url),
        method: _.clone(request.method)
      },
      parsedHeaders = __.sortObjectDeep(formatHeaders(request.headers));

    if (!_.isEmpty(parsedHeaders)) {
      searchNeedle.headers = parsedHeaders;
    }
<commit_msg>feat(core): Included the filter functionallity previously described.<commit_after>'use strict';
/* eslint-env es6 */
/* eslint-disable no-unused-vars */

const Q = require('q');
const Hoxy = require('hoxy');
const Mongoose = require('mongoose');
Mongoose.Promise = require('q').Promise;
const Prettyjson = require('prettyjson');
const _ = require('lodash');
const __ = require('./lodash-like.js');
const Child_process = require('child_process');
var mongoDB;

exports.serve = function(configParams) {
  var servicesShemaModel =
    {
      url: String,
      method: String,
      headers: Object,
      body: Object,
      responseData: Object,
      responseStatusCode: Number,
      timeStamp: Date
    },
    ServicesSchema, proxy;

  function formatHeaders(headers) {
    return _.omit(headers, ['host', 'user-agent', 'accept', 'accept-language', 'accept-encoding', 'dnt',
      'x-requested-with', 'content-type', 'referer', 'content-length', 'cookie', 'origin',
      'connection', 'pragma', 'cache-control']);
  }

  function respondEntry(response, entry) {
    var responseData = entry.responseData || entry.successData; // <- legacy support.
    response.statusCode = entry.responseStatusCode || 200;
    response.string = JSON.stringify(responseData);
  }

  function updateRequestEntry(request, response, entry) {
    entry.responseData = response.json;
    return entry;
  }

  function createRequestEntry(request, response) {
    (new ServicesSchema({
      url: request.url,
      method: request.method,
      headers: __.sortObjectDeep(formatHeaders(request.headers), true),
      body: __.sortObjectDeep(request.json, true),
      responseData: response.json,
      responseStatusCode: response.statusCode,
      timeStamp: Date.now()
    })).save(function(error) {
      if (error) return console.error(error);
    });
  }

  function responseInterceptor(request, response) {
    if (configParams.filter &&
    Child_process.spawnSync(configParams.filter, [JSON.stringify(response.json)]).status === 0) {
      createRequestEntry(request, response);
    }

    if (!configParams.quiet) {
      requestPrintLog(request);
    }
  }

  function printSearchNeedle(searchNeedle) {
    console.info('===================== Serach Needle =======================');
    console.info(Prettyjson.render({searchedTimeStamp: String(Date.now())}));
    console.info();
    console.info(Prettyjson.render(__.sortObjectDeep(searchNeedle.source, true)));
  }

  function requestPrintLog(request) {
    console.info('============ Request from greenTomato -> server ===============');
    console.info(Prettyjson.render({requestTimeStamp: String(Date.now())}));
    console.info();
    console.info(Prettyjson.render(request));
  }

  function getSearchNeedle(request) {
    var searchNeedle =
      {
        url: _.clone(request.url),
        method: _.clone(request.method)
      },
      parsedHeaders = __.sortObjectDeep(formatHeaders(request.headers));

    if (!_.isEmpty(parsedHeaders)) {
      searchNeedle.headers = parsedHeaders;
    }

{'fraction_non_alphanumeric': 0.16046039268788084, 'fraction_numerical': 0.0015233581584292485, 'mean_word_length': 3.8754125412541254, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5062652', 'n_tokens_mistral': 1739, 'n_tokens_neox': 1699, 'n_words': 330}