<commit_before>var request = require('superagent');
var EJSON = require('mongodb-extended-json');

// Override superagent's parse and encode handlers rather than
// adding another content-type tooling doesnt like.
request.serialize['application/json'] = EJSON.stringify;
request.parse['application/json'] = EJSON.parse;

if (typeof window === 'undefined') {
  request.parse['application/json'] = function(res, fn) {
    res.text = '';
    res.setEncoding('utf8');
    res.on('data', function(chunk) {<commit_msg>INT-1193 Fix "Unexpected token o" error<commit_after>var request = require('superagent');
var EJSON = require('mongodb-extended-json');

// Override superagent's parse and encode handlers rather than
// adding another content-type tooling doesnt like.
request.serialize['application/json'] = EJSON.stringify;

var parseJSON = function parseJSON(res, fn) {
  res.text = '';
  res.setEncoding('utf8');
  res.on('data', function(chunk) { res.text += chunk;});
  res.on('end', function() {
    var err;
    var body;
    try {
      body = res.text && EJSON.parse(res.text);
    } catch (e) {
      err = e;
      err.rawResponse = res.text || null;
      err.statusCode = res.statusCode;
    } finally {
      fn(err, body);
    }
  });
};

request.parse['application/json'] = parseJSON;

if (typeof window === 'undefined') {
  request.parse['application/json'] = function(res, fn) {
    res.text = '';
    res.setEncoding('utf8');
    res.on('data', function(chunk) {
{'fraction_non_alphanumeric': 0.16316858496953285, 'fraction_numerical': 0.004739336492890996, 'mean_word_length': 3.8940397350993377, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1402856', 'n_tokens_mistral': 466, 'n_tokens_neox': 453, 'n_words': 108}