<commit_before>/* jshint node: true, browser: true */
/* global EventSource */
var util = require('util');
var Duplex = require('stream').Duplex;

// AJAXStream constructor
function AJAXStream(endpointURL) {
  Duplex.call(this, {objectMode: true});
  this._endpointURL = endpointURL;
  this._messagesQueue = [];
  this._initEventSource();
}

util.inherits(AJAXStream, Duplex);

// expose AJAXStream class
module.exports = AJAXStream;

// Create convenience alias
var p = AJAXStream.prototype;

p._write = function(message, encoding, cb) {
  try {
    this._send(message);
    return cb();
  } catch(err) {
    return cb(err);
  }
};

p._read = function() {
  var message;
  var queue = this._messagesQueue;
  var keepPushing = true;
  while(queue.length > 0 && keepPushing) {
    message = queue.shift();
    keepPushing = this.push(message);
  }
};

p._send = function(message) {
  var self = this;
  var url = self._endpointURL;
  self.emit('send', message);
  $.post(url, message, function(message) {
    self.emit('complete', message);
    if(message) {
      self._enqueue(message);
    }
  });
};

p._enqueue = function(message) {
  this._messagesQueue.push(message);
  this._read();
};

p._initEventSource = function() {
  var eventSource = this._eventSource = new EventSource(this._endpointURL);
  eventSource.addEventListener('message', this._handleSSE.bind(this));
};

p._handleSSE = function(evt) {
  var message;
  try {
    message = JSON.parse(evt.data);
  } catch(err) { /* Ignore invalid message */ }
  if(message) {
    this._enqueue(message);<commit_msg>Proper JSON serialization in AJAXStream<commit_after>/* jshint node: true, browser: true */
/* global EventSource */
var util = require('util');
var Duplex = require('stream').Duplex;

// AJAXStream constructor
function AJAXStream(endpointURL) {
  Duplex.call(this, {objectMode: true});
  this._endpointURL = endpointURL;
  this._messagesQueue = [];
  this._initEventSource();
}

util.inherits(AJAXStream, Duplex);

// expose AJAXStream class
module.exports = AJAXStream;

// Create convenience alias
var p = AJAXStream.prototype;

p._write = function(message, encoding, cb) {
  try {
    this._send(message);
    return cb();
  } catch(err) {
    return cb(err);
  }
};

p._read = function() {
  var message;
  var queue = this._messagesQueue;
  var keepPushing = true;
  while(queue.length > 0 && keepPushing) {
    message = queue.shift();
    keepPushing = this.push(message);
  }
};

p._send = function(message) {
  var self = this;
  var url = self._endpointURL;
  self.emit('send', message);
  $.ajax({
    type: 'POST',
    url: url, 
    data: JSON.stringify(message),
    contentType: 'application/json',
    success: function(message) {
      self.emit('complete', message);
      if(message) {
        self._enqueue(message);
      }
    }
  });
};

p._enqueue = function(message) {
  this._messagesQueue.push(message);
  this._read();
};

p._initEventSource = function() {
  var eventSource = this._eventSource = new EventSource(this._endpointURL);
  eventSource.addEventListener('message', this._handleSSE.bind(this));
};

p._handleSSE = function(evt) {
  var message;
  try {
    message = JSON.parse(evt.data);
  } catch(err) { /* Ignore invalid message */ }
  if(message) {
    this._enqueue(message);
{'fraction_non_alphanumeric': 0.15045592705167174, 'fraction_numerical': 0.0006079027355623101, 'mean_word_length': 3.7626628075253254, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3189713', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1076, 'n_words': 229}