<commit_before>const reqr = global.GENTLY ? GENTLY.hijack(require) : require
const got = reqr('got')
const FormData = require('form-data')
const crypto = reqr('crypto')
const _ = reqr('underscore')
const fs = reqr('fs')
const path = reqr('path')
const retry = reqr('retry')
const PaginationStream = reqr('./PaginationStream')
const Readable = reqr('stream').Readable
const tus = reqr('tus-js-client')
const { access } = reqr('fs').promises

const version = reqr('../package.json').version

function unknownErrMsg (str) {
  let buff = 'Unknown error'
  if (str) {
    buff += ` ${str}`
  }
  buff += '. Please report this at '
  buff += 'https://github.com/transloadit/node-sdk/issues/new?title=Unknown%20error'
  return buff
}

// @todo support size retrieval for other streams
function canGetStreamSizes (streams) {
  for (const stream of streams) {
    // the request module has path attribute that is different from file path
    // but it also has the attribute httpModule
    if (!(stream.path && !stream.httpModule)) {
      return false
    }
  }

  return true
}

class TransloaditClient {
  constructor (opts = {}) {
    if (opts.useSsl == null) {
      opts.useSsl = true
    }

    if (opts.authKey == null) {
      throw new Error('Please provide an authKey')
    }

    if (opts.authSecret == null) {
      throw new Error('Please provide an authSecret')
    }

    this._authKey = opts.authKey
    this._authSecret = opts.authSecret
    this._service = opts.service || 'api2.transloadit.com'
    this._protocol = opts.useSsl ? 'https://' : 'http://'
    this._streams = {}

    this._lastUsedAssemblyUrl = ''
  }

  /**
   * Adds an Assembly file stream
   *
   * @param {string} name fieldname of the file
   * @param {ReadableStream} stream stream to be uploaded
   */
  addStream (name, stream) {
    stream.pause()
    this._streams[name] = stream
  }

  /**
   * Adds an Assembly file
   *
   * @param {string} name field name of the file
   * @param {string} path path to the file
   */
  addFile (name, path) {
    const stream = fs.createReadStream(path)
    stream.on('error', err => {
      // handle the error event to avoid the error being thrown
      console.error(err)
    })
    this.addStream(name, stream)
  }

  getLastUsedAssemblyUrl () {
    return this._lastUsedAssemblyUrl
  }

  /**
   * Create an Assembly
   *
   * @typedef {object} progressObject
   * @property {object} assemblyProgress
   * @property {{totalBytes: number, uploadedBytes: number}} uploadProgress
   *
   * @callback progressCb
   * @param {progressObject} progress
   *
   * @param {object} opts assembly options
   * @param {progressCb} progressCb callback function to be triggered as on each progress update of the assembly
   * @returns {Promise}
   */
  async createAssemblyAsync (opts, progressCb) {
    const defaultOpts = {
      params           : {},
      fields           : {},
      waitForCompletion: false,
      isResumable      : true,
    }
    opts = { ...defaultOpts, ...opts }

    this._lastUsedAssemblyUrl = `${this._serviceUrl()}/assemblies`

    const requestOpts = {
      url    : this._lastUsedAssemblyUrl,
      method : 'post',
      timeout: 24 * 60 * 60 * 1000, // 1 day
      params : opts.params,
      fields : opts.fields,
    }

    let streamsMap = this._streams
    let streams = Object.values(streamsMap)

    // reset streams so they do not get used again in subsequent requests
    this._streams = {}

    // TODO imrpvoe all this
    const useTus = opts.isResumable && canGetStreamSizes(streams)
    const tusStreamsMap = useTus ? streamsMap : {}
    if (useTus) {
      requestOpts.tus_num_expected_upload_files = streams.length
      // make sure they don't get uploaded as multipart (will use tus instead)
      streamsMap = {}
      streams = []
    } else if (opts.isResumable) {
      opts.isResumable = false
      console.warn('disabling resumability because the size of one or more streams cannot be determined')
    }

    // If any stream emits error, we exit with error
    const streamErrorPromise = new Promise((resolve, reject) => {
      streams.forEach((stream) => stream.on('error', reject))
    })

    const mainPromise = (async () => {
      for (const stream of streams) {
        // because an http response stream could also have a "path"
        // attribute but not referring to the local file system
        // see https://github.com/transloadit/node-sdk/pull/50#issue-261982855
        if (!stream.path == null && stream instanceof Readable) {
          await access(stream.path, fs.F_OK | fs.R_OK)
        }
      }

      const result = await this._remoteJson(requestOpts, streamsMap)

      if (result.error != null) throw new Error(result.error)

      if (useTus && Object.keys(tusStreamsMap).length > 0) {
        await this._sendTusRequest(tusStreamsMap, { waitForCompletion: opts.waitForCompletion, assembly: result }, progressCb)
      }

      if (!opts.waitForCompletion) return result
      return this.awaitAssemblyCompletion(result.assembly_id, progressCb)
    })()

    return Promise.race([mainPromise, streamErrorPromise])
  }

  async awaitAssemblyCompletion (assemblyId, progressCb) {
    const result = await this.getAssemblyAsync(assemblyId)
    if (result.error != null) throw new Error(result.error)

    if (result.ok === 'ASSEMBLY_COMPLETED') return result

    if (result.ok === 'ASSEMBLY_UPLOADING' || result.ok === 'ASSEMBLY_EXECUTING') {
      if (progressCb) progressCb({ assemblyProgress: result })

      await new Promise((resolve) => setTimeout(resolve, 1 * 1000))
      // Recurse
      return this.awaitAssemblyCompletion(assemblyId, progressCb)
    }

    throw new Error(unknownErrMsg(`while processing Assembly ID ${assemblyId}`))
  }

  /**
   * Delete the assembly
   *
   * @param {string} assemblyId assembly ID
   * @returns {Promise} after the assembly is deleted
   */
  async deleteAssemblyAsync (assemblyId) {
    // eslint-disable-next-line camelcase
    const { assembly_url } = this.getAssembly(assemblyId) || {}

    const opts = {
      url    : assembly_url,
      timeout: 5000,
      method : 'delete',
    }

    return this._remoteJson(opts)
  }
<commit_msg>fix bug<commit_after>const reqr = global.GENTLY ? GENTLY.hijack(require) : require
const got = reqr('got')
const FormData = require('form-data')
const crypto = reqr('crypto')
const _ = reqr('underscore')
const fs = reqr('fs')
const path = reqr('path')
const retry = reqr('retry')
const PaginationStream = reqr('./PaginationStream')
const Readable = reqr('stream').Readable
const tus = reqr('tus-js-client')
const { access } = reqr('fs').promises

const version = reqr('../package.json').version

function unknownErrMsg (str) {
  let buff = 'Unknown error'
  if (str) {
    buff += ` ${str}`
  }
  buff += '. Please report this at '
  buff += 'https://github.com/transloadit/node-sdk/issues/new?title=Unknown%20error'
  return buff
}

// @todo support size retrieval for other streams
function canGetStreamSizes (streams) {
  for (const stream of streams) {
    // the request module has path attribute that is different from file path
    // but it also has the attribute httpModule
    if (!(stream.path && !stream.httpModule)) {
      return false
    }
  }

  return true
}

class TransloaditClient {
  constructor (opts = {}) {
    if (opts.useSsl == null) {
      opts.useSsl = true
    }

    if (opts.authKey == null) {
      throw new Error('Please provide an authKey')
    }

    if (opts.authSecret == null) {
      throw new Error('Please provide an authSecret')
    }

    this._authKey = opts.authKey
    this._authSecret = opts.authSecret
    this._service = opts.service || 'api2.transloadit.com'
    this._protocol = opts.useSsl ? 'https://' : 'http://'
    this._streams = {}

    this._lastUsedAssemblyUrl = ''
  }

  /**
   * Adds an Assembly file stream
   *
   * @param {string} name fieldname of the file
   * @param {ReadableStream} stream stream to be uploaded
   */
  addStream (name, stream) {
    stream.pause()
    this._streams[name] = stream
  }

  /**
   * Adds an Assembly file
   *
   * @param {string} name field name of the file
   * @param {string} path path to the file
   */
  addFile (name, path) {
    const stream = fs.createReadStream(path)
    stream.on('error', err => {
      // handle the error event to avoid the error being thrown
      console.error(err)
    })
    this.addStream(name, stream)
  }

  getLastUsedAssemblyUrl () {
    return this._lastUsedAssemblyUrl
  }

  /**
   * Create an Assembly
   *
   * @typedef {object} progressObject
   * @property {object} assemblyProgress
   * @property {{totalBytes: number, uploadedBytes: number}} uploadProgress
   *
   * @callback progressCb
   * @param {progressObject} progress
   *
   * @param {object} opts assembly options
   * @param {progressCb} progressCb callback function to be triggered as on each progress update of the assembly
   * @returns {Promise}
   */
  async createAssemblyAsync (opts, progressCb) {
    const defaultOpts = {
      params           : {},
      fields           : {},
      waitForCompletion: false,
      isResumable      : true,
    }
    opts = { ...defaultOpts, ...opts }

    this._lastUsedAssemblyUrl = `${this._serviceUrl()}/assemblies`

    const requestOpts = {
      url    : this._lastUsedAssemblyUrl,
      method : 'post',
      timeout: 24 * 60 * 60 * 1000, // 1 day
      params : opts.params,
      fields : opts.fields,
    }

    let streamsMap = this._streams
    let streams = Object.values(streamsMap)

    // reset streams so they do not get used again in subsequent requests
    this._streams = {}

    // TODO imrpvoe all this
    const useTus = opts.isResumable && canGetStreamSizes(streams)
    const tusStreamsMap = useTus ? streamsMap : {}
    if (useTus) {
      requestOpts.tus_num_expected_upload_files = streams.length
      // make sure they don't get uploaded as multipart (will use tus instead)
      streamsMap = {}
      streams = []
    } else if (opts.isResumable) {
      opts.isResumable = false
      console.warn('disabling resumability because the size of one or more streams cannot be determined')
    }

    // If any stream emits error, we exit with error
    const streamErrorPromise = new Promise((resolve, reject) => {
      streams.forEach((stream) => stream.on('error', reject))
    })

    const mainPromise = (async () => {
      for (const stream of streams) {
        // because an http response stream could also have a "path"
        // attribute but not referring to the local file system
        // see https://github.com/transloadit/node-sdk/pull/50#issue-261982855
        if (!stream.path == null && stream instanceof Readable) {
          await access(stream.path, fs.F_OK | fs.R_OK)
        }
      }

      const result = await this._remoteJson(requestOpts, streamsMap)

      if (result.error != null) throw new Error(result.error)

      if (useTus && Object.keys(tusStreamsMap).length > 0) {
        await this._sendTusRequest(tusStreamsMap, { waitForCompletion: opts.waitForCompletion, assembly: result }, progressCb)
      }

      if (!opts.waitForCompletion) return result
      return this.awaitAssemblyCompletion(result.assembly_id, progressCb)
    })()

    return Promise.race([mainPromise, streamErrorPromise])
  }

  async awaitAssemblyCompletion (assemblyId, progressCb) {
    const result = await this.getAssemblyAsync(assemblyId)
    if (result.error != null) throw new Error(result.error)

    if (result.ok === 'ASSEMBLY_COMPLETED') return result

    if (result.ok === 'ASSEMBLY_UPLOADING' || result.ok === 'ASSEMBLY_EXECUTING') {
      if (progressCb) progressCb({ assemblyProgress: result })

      await new Promise((resolve) => setTimeout(resolve, 1 * 1000))
      // Recurse
      return this.awaitAssemblyCompletion(assemblyId, progressCb)
    }

    throw new Error(unknownErrMsg(`while processing Assembly ID ${assemblyId}`))
  }

  /**
   * Delete the assembly
   *
   * @param {string} assemblyId assembly ID
   * @returns {Promise} after the assembly is deleted
   */
  async deleteAssemblyAsync (assemblyId) {
    // eslint-disable-next-line camelcase
    const { assembly_url } = await this.getAssemblyAsync(assemblyId) || {}

    const opts = {
      url    : assembly_url,
      timeout: 5000,
      method : 'delete',
    }

    return this._remoteJson(opts)
  }

{'fraction_non_alphanumeric': 0.11536910044372731, 'fraction_numerical': 0.005647438483259379, 'mean_word_length': 2.891993720565149, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2193856', 'n_tokens_mistral': 3798, 'n_tokens_neox': 3709, 'n_words': 1044}