<commit_before>import concat = require('concat-stream')
import FormData = require('form-data')
import { createUnzip } from 'zlib'
import Promise = require('native-or-bluebird')
export * from './common'
import { headers as defaultHeaders, parse, stringify } from './common'
import Request, { Middleware } from '../request'
import Response from '../response'

function unzipResponse (response: Response) {
  if (['gzip', 'deflate'].indexOf(response.get('Content-Encoding')) > -1) {
    const unzip = createUnzip()
    response.body.pipe(unzip)
    response.body = unzip
  }
}

/**
 * Automatically unzip node HTTP responses.
 */
export function unzip (request: Request) {
  request.after(unzipResponse)
}

/**
 * Create a built-in concat stream plugin for node.
 */
export function concatStream (encoding: string) {
  return function concatStream (request: Request) {
    request.after(function (response: Response) {
      return new Promise(function (resolve, reject) {
        const stream = concat({
          encoding: encoding
        }, function (data: any) {
          // Update the response `body` to the concat output.
          response.body = data
          return resolve()
        })

        response.body.once('error', reject)
        response.body.pipe(stream)
      })
    })
  }
}

/**
 * Set up default headers for node.
 */
function defaultHeadersNode (request: Request) {
  // Specify a default user agent in node.
  if (!request.get('User-Agent')) {
    request.set('User-Agent', 'https://github.com/blakeembrey/popsicle')
  }

  // Accept zipped responses.
  if (!request.get('Accept-Encoding')) {
    request.set('Accept-Encoding', 'gzip,deflate')
  }

  // Manually set the `Content-Length` and `Content-Type` headers from the
  // form data object because we need to handle boundaries and streams.
  if (request.body instanceof FormData) {
    request.set('Content-Type', 'multipart/form-data; boundary=' + request.body.getBoundary())

    // Asynchronously compute the content length.
    return new Promise(function (resolve, reject) {
      request.body.getLength(function (err: Error, length: number) {
        if (err) {
          request.set('Transfer-Encoding', 'chunked')
        } else {
          request.set('Content-Length', String(length))
        }

        return resolve()
      })
    })
  }

  let length = 0
  const body = request.body

  // Attempt to manually compute the content length.
  if (body && !request.get('Content-Length')) {
    if (Array.isArray(body)) {
      for (let i = 0; i < body.length; i++) {
        length += body[i].length
      }
    } else if (typeof body === 'string') {
      length = Buffer.byteLength(body)
    } else {
      length = body.length
    }

    if (length) {
      request.set('Content-Length', String(length))
    } else if (typeof body.pipe === 'function') {
      request.set('Transfer-Encoding', 'chunked')
    } else {
      return Promise.reject(request.error('Argument error, `options.body`', 'EBODY'))
    }
  }
}

export function headers (request: Request) {
  // Use the request from common.
  defaultHeaders(request)

  // Always use node default headers.
  request.before(defaultHeadersNode)
}

export const defaults: Middleware[] = [stringify, headers, unzip, concatStream('string'), parse]<commit_msg>Add GZIP encoding header with unzip middleware<commit_after>import concat = require('concat-stream')
import FormData = require('form-data')
import { createUnzip } from 'zlib'
import Promise = require('native-or-bluebird')
export * from './common'
import { headers as commonHeaders, parse, stringify } from './common'
import Request, { Middleware } from '../request'
import Response from '../response'

function unzipResponse (response: Response) {
  if (['gzip', 'deflate'].indexOf(response.get('Content-Encoding')) > -1) {
    const unzip = createUnzip()
    response.body.pipe(unzip)
    response.body = unzip
  }
}

function unzipHeaders (request: Request) {
  if (!request.get('Accept-Encoding')) {
    request.set('Accept-Encoding', 'gzip,deflate')
  }
}

/**
 * Support gzipped responses.
 */
export function unzip (request: Request) {
  request.before(unzipHeaders)
  request.after(unzipResponse)
}

/**
 * The body is normally a stream in node, this turns it into a string for browser
 * compatibility (and honestly just making it easier to use).
 */
export function concatStream (encoding: string) {
  return function concatStream (request: Request) {
    request.after(function (response: Response) {
      return new Promise(function (resolve, reject) {
        const stream = concat({
          encoding: encoding
        }, function (data: any) {
          // Update the response `body` to the concat output.
          response.body = data
          return resolve()
        })

        response.body.once('error', reject)
        response.body.pipe(stream)
      })
    })
  }
}

function defaultHeaders (request: Request) {
  // Specify a default user agent in node.
  if (!request.get('User-Agent')) {
    request.set('User-Agent', 'https://github.com/blakeembrey/popsicle')
  }

  // Manually set the `Content-Length` and `Content-Type` headers from the
  // form data object because we need to handle boundaries and streams.
  if (request.body instanceof FormData) {
    request.set('Content-Type', 'multipart/form-data; boundary=' + request.body.getBoundary())

    // Asynchronously compute the content length.
    return new Promise(function (resolve, reject) {
      request.body.getLength(function (err: Error, length: number) {
        if (err) {
          request.set('Transfer-Encoding', 'chunked')
        } else {
          request.set('Content-Length', String(length))
        }

        return resolve()
      })
    })
  }

  let length = 0
  const body = request.body

  // Attempt to manually compute the content length.
  if (body && !request.get('Content-Length')) {
    if (Array.isArray(body)) {
      for (let i = 0; i < body.length; i++) {
        length += body[i].length
      }
    } else if (typeof body === 'string') {
      length = Buffer.byteLength(body)
    } else {
      length = body.length
    }

    if (length) {
      request.set('Content-Length', String(length))
    } else if (typeof body.pipe === 'function') {
      request.set('Transfer-Encoding', 'chunked')
    } else {
      return Promise.reject(request.error('Argument error, `options.body`', 'EBODY'))
    }
  }
}

/**
 * Fill default headers with requests (automatic "Content-Length" and "User-Agent").
 */
export function headers (request: Request) {
  commonHeaders(request)
  request.before(defaultHeaders)
}

export const defaults: Middleware[] = [stringify, headers, unzip, concatStream('string'), parse]
{'fraction_non_alphanumeric': 0.13582998959726555, 'fraction_numerical': 0.0008916629514043691, 'mean_word_length': 3.0787878787878786, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3786198', 'n_tokens_mistral': 1950, 'n_tokens_neox': 1928, 'n_words': 549}