<commit_before>var assert = require("assert")
  , url = require("url")

var request = require("request")
  , once = require("once")

module.exports = fetch

function fetch (uri, headers, cb) {
  assert(uri, "must pass resource to fetch")
  assert(cb, "must pass callback")

  if (!headers) headers = {}

  cb = once(cb)

  var client = this
  this.attempt(function (operation) {
    makeRequest.call(client, uri, headers, function (er, req) {
      if (er) return cb(er)

      req.on("error", function (err) {
        if (operation.retry(err)) {
          client.log.info("retry", "will retry, error on last attempt: " + err)
        }
      })

      req.on("response", function (res) {
        client.log.http("fetch", res.statusCode, uri)

        // Only retry on 408, 5xx or no `response`.
        var statusCode = res && res.statusCode

        var timeout = statusCode === 408
        if (timeout) er = new Error("request timed out")

        var serverError = statusCode >= 500
        if (serverError) er = new Error("server error " + statusCode)

        if (er && operation.retry(er)) {
          client.log.info("retry", "will retry, error on last attempt: " + er)
          return
        }

        // Work around bug in node v0.10.0 where the CryptoStream
        // gets stuck and never starts reading again.
        res.resume()
        if (process.version === "v0.10.0") unstick(res)

        cb(null, res)
      })<commit_msg>remove unsightly quotes in logs (fixes #58)<commit_after>var assert = require("assert")
  , url = require("url")

var request = require("request")
  , once = require("once")

module.exports = fetch

function fetch (uri, headers, cb) {
  assert(uri, "must pass resource to fetch")
  assert(cb, "must pass callback")

  if (!headers) headers = {}

  cb = once(cb)

  var client = this
  this.attempt(function (operation) {
    makeRequest.call(client, uri, headers, function (er, req) {
      if (er) return cb(er)

      req.on("error", function (err) {
        if (operation.retry(err)) {
          client.log.info("retry", "will retry, error on last attempt: " + err)
        }
      })

      req.on("response", function (res) {
        client.log.http("fetch", "" + res.statusCode, uri)

        // Only retry on 408, 5xx or no `response`.
        var statusCode = res && res.statusCode

        var timeout = statusCode === 408
        if (timeout) er = new Error("request timed out")

        var serverError = statusCode >= 500
        if (serverError) er = new Error("server error " + statusCode)

        if (er && operation.retry(er)) {
          client.log.info("retry", "will retry, error on last attempt: " + er)
          return
        }

        // Work around bug in node v0.10.0 where the CryptoStream
        // gets stuck and never starts reading again.
        res.resume()
        if (process.version === "v0.10.0") unstick(res)

        cb(null, res)
      })
{'fraction_non_alphanumeric': 0.1359754014349163, 'fraction_numerical': 0.012982576016399043, 'mean_word_length': 2.4006968641114983, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3770525', 'n_tokens_mistral': 912, 'n_tokens_neox': 869, 'n_words': 276}