<commit_before>var router = require('express').Router();
var routes = {};
var _ = require('lodash');
var request = require('request');
var events = require('events').EventEmitter;
var ServerEvents = new events();

// Set maximum event listeners to unlimited.
ServerEvents.setMaxListeners(0)

var eventData = {};
eventData.triggerCount = 0;
eventData.clientCount = 0;

var redUrls = [
  'https://www.reddit.com/r/node.json',
  'https://www.reddit.com/r/javascript.json',
  'https://www.reddit.com/r/webdev.json'
];

// Using Express Router here.

module.exports = function expressRoutes(app){

  router.get('/', routes.index);
  router.get('/trigger', routes.trigger);
  router.get('/complex', routes.complex)
  router.get('/event-stream', routes.events);
  app.use('/', router);

  return router
};

// Render our main page.

routes.index = function(req, res){
  res.render('index')
};

// Our simple stat gatehering event.

routes.trigger = function(req, res){
  eventData.triggerCount = eventData.triggerCount += 1
  ServerEvents.emit('incoming');
  console.log('Trigger count: ' + eventData.triggerCount)
  res.json({status: "success"});
};

//
// Fetch data from reddit and emit the 'complex' event with the returned json.
//

routes.complex = function(req, res){
  var thisReq = redUrls.pop()
  redUrls.unshift(thisReq)
  request(thisReq, function(err, resp, body){
    var posts = JSON.parse(body);
    var data = JSON.stringify({url: thisReq, posts: posts})
    ServerEvents.emit('complex', data)
    res.json({status: "success"})
  })

};

//
// Event streamer, maintains a persistent connection with the browser.
// Content-Type is 'text/event-stream'
//
// Note: depending on the browser, you will only be able to open a limited number of
// Persistent connections to your server on the same domain. In chrome, I found that I could open
// 5 instances of the page before things went haywire, while the limit is technically 6, on the 6th page
// the browser no longer has any available connections to load page resources with.
//

routes.events = function(req, res){
  eventData.clientCount = eventData.clientCount += 1
  console.log('Client Count: ' + eventData.clientCount)
  req.socket.setTimeout(60 * 60 * 1000);
  res.writeHead(200, {
    'Content-Type': 'text/event-stream',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive'
  });
  res.write('\n')

  // Store a reference to out event handler so we can bind and unbind it
  // in the correct context. Otherwise it will fire for every client that has ever connected to
  // this event stream.
  var handleComplex = function(data){
    res.write('event: complex\n');
    res.write('data: ' + data + '\n\n');
  };

  var handleIncoming = function(){
    eventData.memUsage = process.memoryUsage()
    // event: allows you to bind to a specific event on the client side.
    // This allows you to handle multiple event streams easily from one endpoint.
    res.write('event: incoming\n');
    res.write('data: ' + JSON.stringify(eventData) + '\n\n');
  };

  //
  // Bind our event handlers. Note that it is possible to listen for many different events
  // here, within reason. I dont think stuffing 25 different events into a single endpoint
  // Is appropriate.
  //

  ServerEvents.addListener('incoming', handleIncoming)
  ServerEvents.addListener('complex', handleComplex)
<commit_msg>Added note about flushing writes when gzip is used<commit_after>var router = require('express').Router();
var routes = {};
var _ = require('lodash');
var request = require('request');
var events = require('events').EventEmitter;
var ServerEvents = new events();

// Set maximum event listeners to unlimited.
ServerEvents.setMaxListeners(0)

var eventData = {};
eventData.triggerCount = 0;
eventData.clientCount = 0;

var redUrls = [
  'https://www.reddit.com/r/node.json',
  'https://www.reddit.com/r/javascript.json',
  'https://www.reddit.com/r/webdev.json'
];

// Using Express Router here.

module.exports = function expressRoutes(app){

  router.get('/', routes.index);
  router.get('/trigger', routes.trigger);
  router.get('/complex', routes.complex)
  router.get('/event-stream', routes.events);
  app.use('/', router);

  return router
};

// Render our main page.

routes.index = function(req, res){
  res.render('index')
};

// Our simple stat gatehering event.

routes.trigger = function(req, res){
  eventData.triggerCount = eventData.triggerCount += 1
  ServerEvents.emit('incoming');
  console.log('Trigger count: ' + eventData.triggerCount)
  res.json({status: "success"});
};

//
// Fetch data from reddit and emit the 'complex' event with the returned json.
//

routes.complex = function(req, res){
  var thisReq = redUrls.pop()
  redUrls.unshift(thisReq)
  request(thisReq, function(err, resp, body){
    var posts = JSON.parse(body);
    var data = JSON.stringify({url: thisReq, posts: posts})
    ServerEvents.emit('complex', data)
    res.json({status: "success"})
  })

};

//
// Event streamer, maintains a persistent connection with the browser.
// Content-Type is 'text/event-stream'
//
// Note: depending on the browser, you will only be able to open a limited number of
// Persistent connections to your server on the same domain. In chrome, I found that I could open
// 5 instances of the page before things went haywire, while the limit is technically 6, on the 6th page
// the browser no longer has any available connections to load page resources with.
//

routes.events = function(req, res){
  eventData.clientCount = eventData.clientCount += 1
  console.log('Client Count: ' + eventData.clientCount)
  req.socket.setTimeout(60 * 60 * 1000);
  res.writeHead(200, {
    'Content-Type': 'text/event-stream',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive'
  });
  res.write('\n')

  // Store a reference to out event handler so we can bind and unbind it
  // in the correct context. Otherwise it will fire for every client that has ever connected to
  // this event stream.
  var handleComplex = function(data){
    res.write('event: complex\n');
    res.write('data: ' + data + '\n\n');

    //If you are using the compression middleware with express
    //You will need to flush the writes to the browser.
    //res.flush()
  };

  var handleIncoming = function(){
    eventData.memUsage = process.memoryUsage()
    // event: allows you to bind to a specific event on the client side.
    // This allows you to handle multiple event streams easily from one endpoint.
    res.write('event: incoming\n');
    res.write('data: ' + JSON.stringify(eventData) + '\n\n');
  };

  //
  // Bind our event handlers. Note that it is possible to listen for many different events
  // here, within reason. I dont think stuffing 25 different events into a single endpoint
  // Is appropriate.
  //

  ServerEvents.addListener('incoming', handleIncoming)
  ServerEvents.addListener('complex', handleComplex)

{'fraction_non_alphanumeric': 0.1340280793168331, 'fraction_numerical': 0.0060790273556231, 'mean_word_length': 3.687924016282225, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6498320', 'n_tokens_mistral': 2082, 'n_tokens_neox': 2040, 'n_words': 676}