<commit_before>
import Promise from 'bluebird'
import app from 'app'

// this module is typically used to download large
// binary files, so we use the vanilla request lib
// instead of request-promise
import request from 'request'
import progress from 'request-progress'

import noop from './noop'

let default_headers = {
  'User-Agent': `itchio-app/${app.getVersion()}`
}

let self = {
  request: function (opts) {
    let {url, sink, headers = {}, onprogress = noop} = opts
    headers = Object.assign({}, default_headers, headers)

    let req = progress(request.get({ encoding: null, url, headers }))
    let out = req.pipe(sink)

    return new Promise((resolve, reject) => {
      req.on('progress', onprogress)
      req.on('response', (response) => {
        if (!/^2/.test('' + response.statusCode)) {
          reject(`HTTP ${response.statusCode} while accessing ${url}`)
        }
      })
      req.on('error', reject)

      out.on('close', resolve)
    })
  }
}

export default self<commit_msg>util/http: More frequent progress reports<commit_after>
import Promise from 'bluebird'
import app from 'app'

// this module is typically used to download large
// binary files, so we use the vanilla request lib
// instead of request-promise
import request from 'request'
import progress from 'request-progress'

import noop from './noop'

let default_headers = {
  'User-Agent': `itchio-app/${app.getVersion()}`
}

let self = {
  request: function (opts) {
    let {url, sink, headers = {}, onprogress = noop} = opts
    headers = Object.assign({}, default_headers, headers)

    let req = progress(request.get({ encoding: null, url, headers }), {throttle: 25})
    let out = req.pipe(sink)

    return new Promise((resolve, reject) => {
      req.on('progress', onprogress)
      req.on('response', (response) => {
        if (!/^2/.test('' + response.statusCode)) {
          reject(`HTTP ${response.statusCode} while accessing ${url}`)
        }
      })
      req.on('error', reject)

      out.on('close', resolve)
    })
  }
}

export default self
{'fraction_non_alphanumeric': 0.14534883720930233, 'fraction_numerical': 0.001937984496124031, 'mean_word_length': 3.2142857142857144, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5170226', 'n_tokens_mistral': 626, 'n_tokens_neox': 616, 'n_words': 193}