<commit_before>var url = require('url')
var zlib = require('zlib')
var http = require('http')
var https = require('https')
var Netrc = require('netrc')
var saveTo = require('save-to')
var rawBody = require('raw-body')
var proxyAgent = require('proxy-agent')

exports = module.exports = extend()
exports.extend = extend

var redirectStatusCodes = [
  301,
  302,
  303,
  305,
  307,
]

var httpErrorStati = [
  502,
  503,
  504,
]

var httpErrorCodes = [
  'ECONNRESET',
  'ETIMEDOUT',
  'EADDRINFO',
  'ESOCKETTIMEDOUT',
]

function extend(defaults) {
  defaults = defaults || {}
  defaults.retries = defaults.retries || 0
  defaults.redirects = 'redirects' in defaults ? defaults.redirects : 3
  defaults.timeout = defaults.timeout || 1000
  defaults.method = defaults.method || 'GET'
  defaults.gunzip = defaults.gunzip === false ? false : true
  defaults.netrc = Netrc(defaults.netrc)

  return cogent

  function* cogent(uri, options) {
    // options type checking stuff
    if (options === true)
      options = { json: true }
    else if (typeof options === 'string')
      options = { destination: options }<commit_msg>increase default timeout, default agent to false<commit_after>var url = require('url')
var zlib = require('zlib')
var http = require('http')
var https = require('https')
var Netrc = require('netrc')
var saveTo = require('save-to')
var rawBody = require('raw-body')
var proxyAgent = require('proxy-agent')

exports = module.exports = extend()
exports.extend = extend

var redirectStatusCodes = [
  301,
  302,
  303,
  305,
  307,
]

var httpErrorStati = [
  502,
  503,
  504,
]

var httpErrorCodes = [
  'ECONNRESET',
  'ETIMEDOUT',
  'EADDRINFO',
  'ESOCKETTIMEDOUT',
]

function extend(defaults) {
  defaults = defaults || {}
  defaults.retries = defaults.retries || 0
  defaults.redirects = 'redirects' in defaults ? defaults.redirects : 3
  defaults.timeout = defaults.timeout || 5000
  defaults.method = defaults.method || 'GET'
  defaults.gunzip = defaults.gunzip === false ? false : true
  defaults.netrc = Netrc(defaults.netrc)
  defaults.agent = defaults.agent || false

  return cogent

  function* cogent(uri, options) {
    // options type checking stuff
    if (options === true)
      options = { json: true }
    else if (typeof options === 'string')
      options = { destination: options }
{'fraction_non_alphanumeric': 0.1344178082191781, 'fraction_numerical': 0.025684931506849314, 'mean_word_length': 3.808641975308642, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4956882', 'n_tokens_mistral': 795, 'n_tokens_neox': 751, 'n_words': 201}