<commit_before>var request = require("hyperquest");
var through = require('through2');
var user = require("../user");
var mschema = require("mschema");
var streamBuffers = require('stream-buffers');
var through = require('through2');
var Mustache = require("mustache");
var View = require('view').View;
var trycatch = require('trycatch');

module['exports'] = function runUntrustedHook (opts, untrustedHook, cb) {

  var hook = require('./');

  var req = opts.req,
      res = opts.res;

  console.log('running untrusted hook', opts.req.resource.params)
  var untrustedSchema = untrustedHook.schema || {};
  var defaults = mschema.validate(opts.req.resource.params, untrustedHook.schema || {}, { strict: false });
  var debugOutput = [];
   var open = function open (url) {
     return request(url, {
       "headers": {
         "accept": "*/*"
       }
     });
   };

   var post = function post (url) {
     return request.post(url, {
       "headers": {
         "accept": "*/*"
       }
     });
   };

   // A simple debug utility for inspecting data from inside a running hook
   var debug = function debug (arg) {
     debugOutput.push({
       time: new Date(),
       data: arg,
       ip : opts.req.connection.remoteAddress
     })
   };

   // create a new buffer and output stream for capturing the hook.res.write and hook.res.end calls from inside the hook
   // this is used as an intermediary to pipe hook output to other streams ( such as another hook )
   var hookOutput = new streamBuffers.WritableStreamBuffer({
       initialSize: (100 * 1024),        // start as 100 kilobytes.
       incrementAmount: (10 * 1024)    // grow by 10 kilobytes each time buffer overflows.
   });

   var _headers = {
     code: null,
     headers: {}
   };

  var output = through(function (chunk, enc, callback) {
    hookOutput.write(chunk);
    callback()
  }, function(){
    var content = hookOutput.getContents();
    // Apply basic mustache replacements
    var strTheme = opts.theme;
    strTheme = Mustache.render(strTheme, {
      hook: {
        output: content.toString(),
        debug: JSON.stringify(debugOutput, true, 2),
        params: defaults.instance,
        headers: _headers,
        schema: untrustedSchema
      }
    });

    if (req.resource.params.format === "raw") {
      cb(null, {
        output: content, //hookOutput,
        debug: debugOutput,
        params: defaults.instance,
        headers: _headers,
        hook: {
          schema: untrustedSchema
        }
      })
    }

    var _view = new View({ template: strTheme, presenter: opts.presenter });

    // give the presenter 3 seconds to render, or else it has failed
    var completedTimer = setTimeout(function(){
      if (!completed) {
        return cb(new Error('Hook presenter took more than 3 seconds to load. Aborting request. \n\nA delay of this long usually means the presenter never fired it\'s callback. Check the presenter code for error. \n\nIf this is not the case and you require more than 3 seconds to present your view, please contact <EMAIL>'));
      }
    }, 3000);
    var completed = false;

    try { // this will catch user run-time errors in the presenter
      _view.present({
        request: req,
        response: res,
        output: content,
        debug: debugOutput,
        instance: defaults.instance,
        params: opts.req.resource.params,
        headers: _headers
      }, function(err, rendered){
        completed = true;
        completedTimer = clearTimeout(completed);
        cb(null, {
          output: rendered,
          debug: debugOutput,
          params: defaults.instance,
          headers: _headers,
          hook: {
            schema: untrustedSchema
          }
        });
      });
    } catch (err) {
      cb(err);
    }


  });
  var _res = res;

  output.writeHead = function (code, headers) {
    _headers.code = code;
    for(var h in headers) {
      _headers.headers[h] = headers[h];
    }
  };

  if (opts.params.format === "friendly") {
    _res = output;
  }
  
  if (defaults.valid === false) {
   output.writeHead(500);
   output.write(JSON.stringify(defaults.errors, true, 2));
   output.end();
   return cb(defaults.errors, opts, untrustedHook)
   //return hook.postprocessHook(false, opts, untrustedHook);
   //return opts.res.end(JSON.stringify(defaults.errors, true, 2))
  }

  // async try / catch is required for async user errors
  trycatch(function() {
    var isStreaming = false;
    if (req._readableState.buffer && req._readableState.buffer.length) {
      isStreaming = true;
    }
    untrustedHook({
      env: req.env,
      debug: debug,
      get: open,
      open: open,
      post: post,
      params: defaults.instance,
      req: opts.req,
      res: _res,
      streaming: isStreaming
    });
  }, function(err) {
    console.log(err.stack);
    return cb(err, opts, untrustedHook)
  });

}<commit_msg>[fix] Ensure that `hook.postProcessHook` gets called for raw responses #25<commit_after>var request = require("hyperquest");
var through = require('through2');
var user = require("../user");
var mschema = require("mschema");
var streamBuffers = require('stream-buffers');
var through = require('through2');
var Mustache = require("mustache");
var View = require('view').View;
var trycatch = require('trycatch');

module['exports'] = function runUntrustedHook (opts, untrustedHook, cb) {

  var hook = require('./');

  var req = opts.req,
      res = opts.res;

  //
  // TODO: add timestamps to console.log output
  //
  console.log('running untrusted hook', opts.req.resource.params);

  var untrustedSchema = untrustedHook.schema || {};
  var defaults = mschema.validate(opts.req.resource.params, untrustedHook.schema || {}, { strict: false });
  var debugOutput = [];
   var open = function open (url) {
     return request(url, {
       "headers": {
         "accept": "*/*"
       }
     });
   };

   var post = function post (url) {
     return request.post(url, {
       "headers": {
         "accept": "*/*"
       }
     });
   };

   // A simple debug utility for inspecting data from inside a running hook
   var debug = function debug (arg) {
     debugOutput.push({
       time: new Date(),
       data: arg,
       ip : opts.req.connection.remoteAddress
     })
   };

   // create a new buffer and output stream for capturing the hook.res.write and hook.res.end calls from inside the hook
   // this is used as an intermediary to pipe hook output to other streams ( such as another hook )
   var hookOutput = new streamBuffers.WritableStreamBuffer({
       initialSize: (100 * 1024),        // start as 100 kilobytes.
       incrementAmount: (10 * 1024)    // grow by 10 kilobytes each time buffer overflows.
   });

   var _headers = {
     code: null,
     headers: {}
   };

  var output = through(function (chunk, enc, callback) {
    hookOutput.write(chunk);
    callback()
  }, function(){
    var content = hookOutput.getContents();
    // Apply basic mustache replacements
    var strTheme = opts.theme;
    strTheme = Mustache.render(strTheme, {
      hook: {
        output: content.toString(),
        debug: JSON.stringify(debugOutput, true, 2),
        params: defaults.instance,
        headers: _headers,
        schema: untrustedSchema
      }
    });

    var _view = new View({ template: strTheme, presenter: opts.presenter });

    // give the presenter 3 seconds to render, or else it has failed
    var completedTimer = setTimeout(function(){
      if (!completed) {
        return cb(new Error('Hook presenter took more than 3 seconds to load. Aborting request. \n\nA delay of this long usually means the presenter never fired it\'s callback. Check the presenter code for error. \n\nIf this is not the case and you require more than 3 seconds to present your view, please contact <EMAIL>'));
      }
    }, 3000);
    var completed = false;

    try { // this will catch user run-time errors in the presenter
      _view.present({
        request: req,
        response: res,
        output: content,
        debug: debugOutput,
        instance: defaults.instance,
        params: opts.req.resource.params,
        headers: _headers
      }, function(err, rendered){
        completed = true;
        completedTimer = clearTimeout(completed);
        cb(null, {
          output: rendered,
          debug: debugOutput,
          params: defaults.instance,
          headers: _headers,
          hook: {
            schema: untrustedSchema
          }
        });
      });
    } catch (err) {
      cb(err);
    }


  });
  var _res = res;

  output.writeHead = function (code, headers) {
    _headers.code = code;
    for(var h in headers) {
      _headers.headers[h] = headers[h];
    }
  };

  if (opts.params.format === "friendly") {
    _res = output;
  }
  
  if (defaults.valid === false) {
   output.writeHead(500);
   output.write(JSON.stringify(defaults.errors, true, 2));
   output.end();
   return cb(defaults.errors, opts, untrustedHook)
   //return hook.postprocessHook(false, opts, untrustedHook);
   //return opts.res.end(JSON.stringify(defaults.errors, true, 2))
  }

  // async try / catch is required for async user errors
  trycatch(function() {
    var isStreaming = false;
    if (req._readableState.buffer && req._readableState.buffer.length) {
      isStreaming = true;
    }
    untrustedHook({
      env: req.env,
      debug: debug,
      get: open,
      open: open,
      post: post,
      params: defaults.instance,
      req: opts.req,
      res: _res,
      streaming: isStreaming
    });
  }, function(err) {
    console.log('Error', opts.req.url, err.stack);
  });

  if (req.resource.params.format === "raw") {
    return cb(null, opts, untrustedHook);
  }

}
{'fraction_non_alphanumeric': 0.12326798727291388, 'fraction_numerical': 0.006979369803961819, 'mean_word_length': 2.4974874371859297, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6945169', 'n_tokens_mistral': 2917, 'n_tokens_neox': 2811, 'n_words': 846}