<commit_before>// Namespace.
var remutil = typeof require == 'undefined' ? this.remutil = {} : exports;

/**
 * Utilities
 */

remutil.clone = function (obj) {
  return JSON.parse(JSON.stringify(obj));
};

remutil.modify = function (a, b) {
  var c = remutil.clone(a);
  for (var k in a) {
    if (Object.prototype.hasOwnProperty.call(a, k)) {
      c[k] = a[k];
    }
  }
  for (var k in b) {
    if (Object.prototype.hasOwnProperty.call(b, k)) {
      c[k] = b[k];
    }
  }
  return c;
};

remutil.callable = function (obj) {
  var f = function () {
    return f.call.apply(f, arguments);
  };
  f.__proto__ = obj;
  return f;
};

remutil.safeJSONStringify = function (s) {
  return JSON.stringify(s).replace(/[\u007f-\uffff]/g, function (c) {
    return "\\u" + ("0000" + c.charCodeAt(0).toString(16)).slice(-4);
  });
};

remutil.inherits = function (ctor, superCtor) {
  ctor.super_ = superCtor;
  ctor.prototype = Object.create(superCtor.prototype, {
    constructor: {
      value: ctor,
      enumerable: false
    }
  });
};

remutil.consumeStream = null;

/**
 * EventEmitter
 */

function EventEmitter () { }

EventEmitter.prototype.listeners = function (type) {
  return this.hasOwnProperty.call(this._events || (this._events = {}), type) ? this._events[type] : this._events[type] = [];
};

EventEmitter.prototype.on = EventEmitter.prototype.addListener = function (type, f) {
  if (this._maxListeners !== 0 && this.listeners(type).push(f) > (this._maxListeners || 10)) {
    console && console.warn('Possible EventEmitter memory leak detected. ' + this._events[type].length + ' listeners added. Use emitter.setMaxListeners() to increase limit.');
  }
  this.emit("newListener", type, f);
  return this;
};

EventEmitter.prototype.removeListener = function (type, f) {
  var i;
  (i = this.listeners(type).indexOf(f)) != -1 && this.listeners(type).splice(i, 1);
  return this;
};

EventEmitter.prototype.removeAllListeners = function (type) {
  for (var k in this._events) {
    (!type || type == k) && this._events[k].splice(0, this._events[k].length);
  }
  return this;
};

EventEmitter.prototype.emit = function (type) {
  var args = Array.prototype.slice.call(arguments, 1);
  for (var i = 0, fns = this.listeners(type).slice(); i < fns.length; i++) {
    fns[i].apply(this, args);
  }
  return fns.length;
};

EventEmitter.prototype.setMaxListeners = function (maxListeners) {
  this._maxListeners = maxListeners;
};

remutil.EventEmitter = EventEmitter;

/** 
 * URL primitive
 */

// protocol://auth@hostname:port/pathname?query#hash

remutil.url = {

  parse: null,

  format: null,

  host: function (url) {
    return url.hostname && (url.hostname + (url.port ? ':' + url.port : ''));
  },

  path: function (url) {
    return url.pathname
      + (url.query ? '?' + remutil.qs.stringify(url.query) : '')
      + (url.hash ? '#' + encodeURIComponent(url.hash) : '');
  }

};

/**
 * Request primitive
 */

remutil.request = {

  create: function (mod) {
    return remutil.request.url({
      method: 'GET',
      headers: {},
      url: null,
      body: null
    }, mod);
  },

  url: function (opts, mod) {
    if (typeof mod == 'string') {
      mod = remutil.url.parse(mod);
    }
    mod.query = remutil.modify(opts.url ? opts.url.query : {}, mod.query);
    opts = remutil.modify(opts, {
      url: opts.url ? remutil.modify(opts.url, mod) : mod
    });
    return remutil.modify(opts, {
      headers: remutil.modify(opts.headers, {
        'Host': remutil.url.host(opts.url)
      })
    })
  },

  body: function (opts, type, body) {
    // Expand payload shorthand.
    if (typeof body == 'object') {
      if (type == 'form' || type == 'application/x-www-form-urlencoded') {
        type = 'application/x-www-form-urlencoded';
        body = remutil.qs.stringify(body);
      }
      if (type == 'json' || type == 'application/json') {
        type = 'application/json';
        body = remutil.safeJSONStringify(body);
      }
    }

    return remutil.modify(opts, {
      headers: remutil.modify(opts.headers, {
        'Content-Type': type,
        'Content-Length': body.length
      }),
      body: body
    });
  },

  send: null

};

/**
 * Query string parsing.
 */

remutil.qs = {

  stringify: null,

  parse: null

};

/**
 * Path
 */

remutil.path = {

  join: null

};

/** 
 * Manifest lookup.
 */

remutil.lookup = function (name) {
  var fs = require('fs');
  var path = require('path');
  try {
    return JSON.parse(fs.readFileSync(path.join(__dirname, '../builtin', name + '.json')));
  } catch (e) {
    return null;
  }
};

/**
 * Node.js
 */

(typeof require != 'undefined') && (function () {

  // Stream.

  remutil.consumeStream = function (stream, next) {
    var buf = [];
    stream.on('data', function (data) {
      buf.push(data);
    });
    stream.on('end', function () {
      next(Buffer.concat(buf));
    });
  };

  // URL

  var url = require('url');

  remutil.url.parse = function (str) {
    var parsed = url.parse(String(str), true);
    return {
      protocol: parsed.protocol,
      auth: parsed.auth,
      hostname: parsed.hostname,
      port: parsed.port,
      pathname: parsed.pathname,
      query: parsed.query || {},
      search: parsed.search,
      hash: parsed.hash
    };
  };

  remutil.url.format = function (str) {
    return url.format(str);
  };

  // Request

  var http = require('http');
  var https = require('https');
  var querystring = require('querystring');

  remutil.request.send = function (opts, next) {
    var req = (opts.url.protocol == 'https:' ? https : http).request({
      method: opts.method,
      headers: opts.headers,
      protocol: opts.url.protocol,
      hostname: opts.url.hostname,
      port: opts.url.port,
      path: remutil.url.path(opts.url)
    });

    // Response.
    req.on('response', function (res) {
      // Attempt to follow Location: headers.
      if (((res.statusCode / 100) | 0) == 3 && res.headers['location']) {
        remutil.request.send(remutil.request.url(opts, res.headers['location']), next);
      } else {
        next && next(null, res);
      }
    });

    // Headers.
    if (opts.body != null) {
      req.write(opts.body);
    }
    req.end();

    return req;
  };

  // Query

  remutil.qs = require('querystring');

  // Path

  remutil.path = require('path');

})();

/**
 * Browser.
 */

(typeof require == 'undefined') && (function () {

  // Stream.

  remutil.consumeStream = function (stream, next) {
    var buf = [];
    stream.on('data', function (data) {
      buf.push(data);
    });
    stream.on('end', function () {
      next(buf.join(''));
    });
  };

  // URL

  remutil.url.parse = function (str) {
    var a = document.createElement('a');
    a.href = str;
    return {
      protocol: a.protocol,
      auth: a.auth,
      hostname: a.hostname,
      port: a.port,
      pathname: a.pathname,
      query: remutil.qs.parse(a.search || ''),
      hash: a.hash && decodeURIComponent(a.hash.substr(1))
    };
  };

  remutil.url.format = function (url) {
    var a = document.createElement('a');
    a.protocol = url.protocol;
    a.auth = url.auth;
    a.hostname = url.hostname;
    a.port = url.port;
    a.pathname = url.pathname;
    a.query = remutil.qs.stringify(url.query);
    a.hash = url.hash;
    return a.href;
  };

  // Query string

  remutil.qs.parse = function (query) {<commit_msg>Port on browser was being added as 0, not null<commit_after>// Namespace.
var remutil = typeof require == 'undefined' ? this.remutil = {} : exports;

/**
 * Utilities
 */

remutil.clone = function (obj) {
  return JSON.parse(JSON.stringify(obj));
};

remutil.modify = function (a, b) {
  var c = remutil.clone(a);
  for (var k in a) {
    if (Object.prototype.hasOwnProperty.call(a, k)) {
      c[k] = a[k];
    }
  }
  for (var k in b) {
    if (Object.prototype.hasOwnProperty.call(b, k)) {
      c[k] = b[k];
    }
  }
  return c;
};

remutil.callable = function (obj) {
  var f = function () {
    return f.call.apply(f, arguments);
  };
  f.__proto__ = obj;
  return f;
};

remutil.safeJSONStringify = function (s) {
  return JSON.stringify(s).replace(/[\u007f-\uffff]/g, function (c) {
    return "\\u" + ("0000" + c.charCodeAt(0).toString(16)).slice(-4);
  });
};

remutil.inherits = function (ctor, superCtor) {
  ctor.super_ = superCtor;
  ctor.prototype = Object.create(superCtor.prototype, {
    constructor: {
      value: ctor,
      enumerable: false
    }
  });
};

remutil.consumeStream = null;

/**
 * EventEmitter
 */

function EventEmitter () { }

EventEmitter.prototype.listeners = function (type) {
  return this.hasOwnProperty.call(this._events || (this._events = {}), type) ? this._events[type] : this._events[type] = [];
};

EventEmitter.prototype.on = EventEmitter.prototype.addListener = function (type, f) {
  if (this._maxListeners !== 0 && this.listeners(type).push(f) > (this._maxListeners || 10)) {
    console && console.warn('Possible EventEmitter memory leak detected. ' + this._events[type].length + ' listeners added. Use emitter.setMaxListeners() to increase limit.');
  }
  this.emit("newListener", type, f);
  return this;
};

EventEmitter.prototype.removeListener = function (type, f) {
  var i;
  (i = this.listeners(type).indexOf(f)) != -1 && this.listeners(type).splice(i, 1);
  return this;
};

EventEmitter.prototype.removeAllListeners = function (type) {
  for (var k in this._events) {
    (!type || type == k) && this._events[k].splice(0, this._events[k].length);
  }
  return this;
};

EventEmitter.prototype.emit = function (type) {
  var args = Array.prototype.slice.call(arguments, 1);
  for (var i = 0, fns = this.listeners(type).slice(); i < fns.length; i++) {
    fns[i].apply(this, args);
  }
  return fns.length;
};

EventEmitter.prototype.setMaxListeners = function (maxListeners) {
  this._maxListeners = maxListeners;
};

remutil.EventEmitter = EventEmitter;

/** 
 * URL primitive
 */

// protocol://auth@hostname:port/pathname?query#hash

remutil.url = {

  parse: null,

  format: null,

  host: function (url) {
    return url.hostname && (url.hostname + (url.port ? ':' + url.port : ''));
  },

  path: function (url) {
    return url.pathname
      + (url.query ? '?' + remutil.qs.stringify(url.query) : '')
      + (url.hash ? '#' + encodeURIComponent(url.hash) : '');
  }

};

/**
 * Request primitive
 */

remutil.request = {

  create: function (mod) {
    return remutil.request.url({
      method: 'GET',
      headers: {},
      url: null,
      body: null
    }, mod);
  },

  url: function (opts, mod) {
    if (typeof mod == 'string') {
      mod = remutil.url.parse(mod);
    }
    mod.query = remutil.modify(opts.url ? opts.url.query : {}, mod.query);
    opts = remutil.modify(opts, {
      url: opts.url ? remutil.modify(opts.url, mod) : mod
    });
    return remutil.modify(opts, {
      headers: remutil.modify(opts.headers, {
        'Host': remutil.url.host(opts.url)
      })
    })
  },

  body: function (opts, type, body) {
    // Expand payload shorthand.
    if (typeof body == 'object') {
      if (type == 'form' || type == 'application/x-www-form-urlencoded') {
        type = 'application/x-www-form-urlencoded';
        body = remutil.qs.stringify(body);
      }
      if (type == 'json' || type == 'application/json') {
        type = 'application/json';
        body = remutil.safeJSONStringify(body);
      }
    }

    return remutil.modify(opts, {
      headers: remutil.modify(opts.headers, {
        'Content-Type': type,
        'Content-Length': body.length
      }),
      body: body
    });
  },

  send: null

};

/**
 * Query string parsing.
 */

remutil.qs = {

  stringify: null,

  parse: null

};

/**
 * Path
 */

remutil.path = {

  join: null

};

/** 
 * Manifest lookup.
 */

remutil.lookup = function (name) {
  var fs = require('fs');
  var path = require('path');
  try {
    return JSON.parse(fs.readFileSync(path.join(__dirname, '../builtin', name + '.json')));
  } catch (e) {
    return null;
  }
};

/**
 * Node.js
 */

(typeof require != 'undefined') && (function () {

  // Stream.

  remutil.consumeStream = function (stream, next) {
    var buf = [];
    stream.on('data', function (data) {
      buf.push(data);
    });
    stream.on('end', function () {
      next(Buffer.concat(buf));
    });
  };

  // URL

  var url = require('url');

  remutil.url.parse = function (str) {
    var parsed = url.parse(String(str), true);
    return {
      protocol: parsed.protocol,
      auth: parsed.auth,
      hostname: parsed.hostname,
      port: parsed.port,
      pathname: parsed.pathname,
      query: parsed.query || {},
      search: parsed.search,
      hash: parsed.hash
    };
  };

  remutil.url.format = function (str) {
    return url.format(str);
  };

  // Request

  var http = require('http');
  var https = require('https');
  var querystring = require('querystring');

  remutil.request.send = function (opts, next) {
    var req = (opts.url.protocol == 'https:' ? https : http).request({
      method: opts.method,
      headers: opts.headers,
      protocol: opts.url.protocol,
      hostname: opts.url.hostname,
      port: opts.url.port,
      path: remutil.url.path(opts.url)
    });

    // Response.
    req.on('response', function (res) {
      // Attempt to follow Location: headers.
      if (((res.statusCode / 100) | 0) == 3 && res.headers['location']) {
        remutil.request.send(remutil.request.url(opts, res.headers['location']), next);
      } else {
        next && next(null, res);
      }
    });

    // Headers.
    if (opts.body != null) {
      req.write(opts.body);
    }
    req.end();

    return req;
  };

  // Query

  remutil.qs = require('querystring');

  // Path

  remutil.path = require('path');

})();

/**
 * Browser.
 */

(typeof require == 'undefined') && (function () {

  // Stream.

  remutil.consumeStream = function (stream, next) {
    var buf = [];
    stream.on('data', function (data) {
      buf.push(data);
    });
    stream.on('end', function () {
      next(buf.join(''));
    });
  };

  // URL

  remutil.url.parse = function (str) {
    var a = document.createElement('a');
    a.href = str;
    return {
      protocol: a.protocol,
      auth: a.auth,
      hostname: a.hostname,
      port: a.port || null,
      pathname: a.pathname,
      query: remutil.qs.parse(a.search || ''),
      hash: a.hash && decodeURIComponent(a.hash.substr(1))
    };
  };

  remutil.url.format = function (url) {
    var a = document.createElement('a');
    a.protocol = url.protocol;
    a.auth = url.auth;
    a.hostname = url.hostname;
    url.port && (a.port = url.port);
    a.pathname = url.pathname;
    a.query = remutil.qs.stringify(url.query);
    a.hash = url.hash;
    return a.href;
  };

  // Query string

  remutil.qs.parse = function (query) {
{'fraction_non_alphanumeric': 0.1764507649794433, 'fraction_numerical': 0.0034373525645346095, 'mean_word_length': 2.6269860669762894, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3539451', 'n_tokens_mistral': 5155, 'n_tokens_neox': 5093, 'n_words': 1039}