<commit_before>var util      = require('util'),
    events    = require("events"),
    http      = require('http'),
    https     = require('https'),
    url       = require('url'),
    qs        = require('qs'),
    multipart = require('./multipartform'),
    zlib      = require('zlib'),
    iconv     = require('iconv-lite');

function mixin(target, source) {
  source = source || {};
  Object.keys(source).forEach(function(key) {
    target[key] = source[key];
  });

  return target;
}

function Request(uri, options) {
  events.EventEmitter.call(this);
  this.url = url.parse(uri);
  this.options = options;
  this.headers = {
    'Accept': '*/*',
    'User-Agent': 'Restler for node.js',
    'Host': this.url.host
  };

  this.headers['Accept-Encoding'] = 'gzip, deflate';

  mixin(this.headers, options.headers || {});

  // set port and method defaults
  if (!this.url.port) this.url.port = (this.url.protocol == 'https:') ? '443' : '80';
  if (!this.options.method) this.options.method = (this.options.data) ? 'POST' : 'GET';
  if (typeof this.options.followRedirects == 'undefined') this.options.followRedirects = true;

  // stringify query given in options of not given in URL
  if (this.options.query && !this.url.query) {
    if (typeof this.options.query == 'object')
      this.url.query = qs.stringify(this.options.query);
    else this.url.query = this.options.query;
  }

  this._applyBasicAuth();

  if (this.options.multipart) {
    this.headers['Content-Type'] = 'multipart/form-data; boundary=' + multipart.defaultBoundary;
    var multipart_size = multipart.sizeOf(this.options.data, multipart.defaultBoundary);
    if (typeof multipart_size === 'number' && multipart_size === multipart_size) {
        this.headers['Content-Length'] = multipart_size;
    }
    else {
        console.log("Building multipart request without Content-Length header, please specify all file sizes");
    }
  } else {
    if (typeof this.options.data == 'object') {
      this.options.data = qs.stringify(this.options.data);
      this.headers['Content-Type'] = 'application/x-www-form-urlencoded';
      this.headers['Content-Length'] = this.options.data.length;
    }
    if (typeof this.options.data == 'string') {
      var buffer = new Buffer(this.options.data, this.options.encoding || 'utf8');
      this.options.data = buffer;
      this.headers['Content-Length'] = buffer.length;
    }
    if (!this.options.data) {
      this.headers['Content-Length'] = 0;
    }
  }

  var proto = (this.url.protocol == 'https:') ? https : http;

  this.request = proto.request({
    host: this.url.hostname,
    port: this.url.port,
    path: this._fullPath(),
    method: this.options.method,
    headers: this.headers,
    rejectUnauthorized: this.options.rejectUnauthorized
  });

  this._makeRequest();
}

util.inherits(Request, events.EventEmitter);

mixin(Request.prototype, {
  _isRedirect: function(response) {
    return ([301, 302, 303, 307].indexOf(response.statusCode) >= 0);
  },
  _fullPath: function() {
    var path = this.url.pathname || '/';
    if (this.url.hash) path += this.url.hash;
    if (this.url.query) path += '?' + this.url.query;
    return path;
  },
  _applyBasicAuth: function() {
    var authParts;

    if (this.url.auth) {
      authParts = this.url.auth.split(':');
      this.options.username = authParts[0];
      this.options.password = authParts[1];
    }

    if (this.options.username && this.options.password !== undefined) {
      var b = new Buffer([this.options.username, this.options.password].join(':'));
      this.headers['Authorization'] = "Basic " + b.toString('base64');
    }
  },
  _responseHandler: function(response) {
    var self = this;

    if (self._isRedirect(response) && self.options.followRedirects) {
      try {
        // 303 should redirect and retrieve content with the GET method
        // http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
        if (response.statusCode === 303) {
            self.url    = url.parse(url.resolve(self.url.href, response.headers['location']));
            self.options.method = 'GET';
            delete self.options.data;
            self._retry();
        } else {
            self.url = url.parse(url.resolve(self.url.href, response.headers['location']));
            self._retry();
            // todo handle somehow infinite redirects
        }
      } catch(err) {
        err.message = 'Failed to follow redirect: ' + err.message;
        self._fireError(err, response);
      }
    } else {
      var body = '';

      response.setEncoding('binary');

      response.on('data', function(chunk) {
        body += chunk;
      });

      response.on('end', function() {
        response.rawEncoded = body;
        self._decode(new Buffer(body, 'binary'), response, function(err, body) {
          if (err) {
            self._fireError(err, response);
            return;
          }
          response.raw = body;
          body = self._iconv(body, response);
          self._encode(body, response, function(err, body) {
            if (err) {
              self._fireError(err, response);
            } else {
              self._fireSuccess(body, response);<commit_msg>Fixed issue with response handling when using browserify in a non-node environment<commit_after>var util      = require('util'),
    events    = require("events"),
    http      = require('http'),
    https     = require('https'),
    url       = require('url'),
    qs        = require('qs'),
    multipart = require('./multipartform'),
    zlib      = require('zlib'),
    iconv     = require('iconv-lite');

function mixin(target, source) {
  source = source || {};
  Object.keys(source).forEach(function(key) {
    target[key] = source[key];
  });

  return target;
}

function Request(uri, options) {
  events.EventEmitter.call(this);
  this.url = url.parse(uri);
  this.options = options;
  this.headers = {
    'Accept': '*/*',
    'User-Agent': 'Restler for node.js',
    'Host': this.url.host
  };

  this.headers['Accept-Encoding'] = 'gzip, deflate';

  mixin(this.headers, options.headers || {});

  // set port and method defaults
  if (!this.url.port) this.url.port = (this.url.protocol == 'https:') ? '443' : '80';
  if (!this.options.method) this.options.method = (this.options.data) ? 'POST' : 'GET';
  if (typeof this.options.followRedirects == 'undefined') this.options.followRedirects = true;

  // stringify query given in options of not given in URL
  if (this.options.query && !this.url.query) {
    if (typeof this.options.query == 'object')
      this.url.query = qs.stringify(this.options.query);
    else this.url.query = this.options.query;
  }

  this._applyBasicAuth();

  if (this.options.multipart) {
    this.headers['Content-Type'] = 'multipart/form-data; boundary=' + multipart.defaultBoundary;
    var multipart_size = multipart.sizeOf(this.options.data, multipart.defaultBoundary);
    if (typeof multipart_size === 'number' && multipart_size === multipart_size) {
        this.headers['Content-Length'] = multipart_size;
    }
    else {
        console.log("Building multipart request without Content-Length header, please specify all file sizes");
    }
  } else {
    if (typeof this.options.data == 'object') {
      this.options.data = qs.stringify(this.options.data);
      this.headers['Content-Type'] = 'application/x-www-form-urlencoded';
      this.headers['Content-Length'] = this.options.data.length;
    }
    if (typeof this.options.data == 'string') {
      var buffer = new Buffer(this.options.data, this.options.encoding || 'utf8');
      this.options.data = buffer;
      this.headers['Content-Length'] = buffer.length;
    }
    if (!this.options.data) {
      this.headers['Content-Length'] = 0;
    }
  }

  var proto = (this.url.protocol == 'https:') ? https : http;

  this.request = proto.request({
    host: this.url.hostname,
    port: this.url.port,
    path: this._fullPath(),
    method: this.options.method,
    headers: this.headers,
    rejectUnauthorized: this.options.rejectUnauthorized
  });

  this._makeRequest();
}

util.inherits(Request, events.EventEmitter);

mixin(Request.prototype, {
  _isRedirect: function(response) {
    return ([301, 302, 303, 307].indexOf(response.statusCode) >= 0);
  },
  _fullPath: function() {
    var path = this.url.pathname || '/';
    if (this.url.hash) path += this.url.hash;
    if (this.url.query) path += '?' + this.url.query;
    return path;
  },
  _applyBasicAuth: function() {
    var authParts;

    if (this.url.auth) {
      authParts = this.url.auth.split(':');
      this.options.username = authParts[0];
      this.options.password = authParts[1];
    }

    if (this.options.username && this.options.password !== undefined) {
      var b = new Buffer([this.options.username, this.options.password].join(':'));
      this.headers['Authorization'] = "Basic " + b.toString('base64');
    }
  },
  _responseHandler: function(response) {
    var self = this;

    if (self._isRedirect(response) && self.options.followRedirects) {
      try {
        // 303 should redirect and retrieve content with the GET method
        // http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
        if (response.statusCode === 303) {
            self.url    = url.parse(url.resolve(self.url.href, response.headers['location']));
            self.options.method = 'GET';
            delete self.options.data;
            self._retry();
        } else {
            self.url = url.parse(url.resolve(self.url.href, response.headers['location']));
            self._retry();
            // todo handle somehow infinite redirects
        }
      } catch(err) {
        err.message = 'Failed to follow redirect: ' + err.message;
        self._fireError(err, response);
      }
    } else {
      var body = '';

      // When using browserify, response.setEncoding is not defined
      if (response.setEncoding !== null)
        response.setEncoding('binary');

      response.on('data', function(chunk) {
        body += chunk;
      });

      response.on('end', function() {
        response.rawEncoded = body;
        self._decode(new Buffer(body, 'binary'), response, function(err, body) {
          if (err) {
            self._fireError(err, response);
            return;
          }
          response.raw = body;
          body = self._iconv(body, response);
          self._encode(body, response, function(err, body) {
            if (err) {
              self._fireError(err, response);
            } else {
              self._fireSuccess(body, response);
{'fraction_non_alphanumeric': 0.15526365526365526, 'fraction_numerical': 0.008505008505008505, 'mean_word_length': 2.6887417218543046, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1301284', 'n_tokens_mistral': 3309, 'n_tokens_neox': 3220, 'n_words': 652}