<commit_before>const async = require('async')
const Polly = require('aws-sdk/clients/polly').Presigner
const debug = require('debug')
const fs = require('fs-extra')
const got = require('got')
const tempfile = require('tempfile')
const fileExtensions = require('./file-extensions')
const { sanitizeOpts } = require('./sanitize-opts')

/**
 * Creates an object containing all the data.
 */
exports.buildInfo = (text, urlCreator, task, opts) => {
  return {
    opts: opts,
    task: task,
    tempfile: tempfile(`.${fileExtensions[opts.format]}`),
    text: text,
    urlcreator: urlCreator
  }
}

/**
 * Calls AWS Polly with the given info.
 */
exports.callAws = (info, i, callback) => {
  const secsPerMin = 60
  const minsInHalfHour = 30
  const halfHour = secsPerMin * minsInHalfHour

  info.task.title = info.task.title.replace(/\d+\//, `${i}/`)

  let url = info.urlcreator({
    LexiconNames: info.opts.lexicon,
    OutputFormat: info.opts.format,
    SampleRate: info.opts['sample-rate'] ? String(info.opts['sample-rate']) : undefined,
    Text: info.text,
    TextType: info.opts.type,
    VoiceId: info.opts.voice
  }, halfHour)

  let error
  debug('callAws')(`Opening output stream to ${info.tempfile}`)
  let outputStream = fs.createWriteStream(info.tempfile)
  outputStream.on('close', () => {
    debug('callAws')('Closing output stream')
    callback(error)
    /* istanbul ignore if */
    if (error) {
      debug('callAws')(`Error during request: ${error.message}`)
      // Get the error message from Amazon.
      try {
        let response = fs.readFileSync(info.tempfile, 'utf8')
        debug('callAws')(`Amazon responded with ${response}`)
        let parsedResponse = JSON.parse(response)
        error.message += `: ${parsedResponse.message}`
      } catch (err) { }
    }
  })
  let sanitizedUrl = url
    .replace(/(Text=.{1,10})([^&]*)/, '$1...')
    .replace(/(X-Amz-Credential=)([^&]*)/, '$1...')
    .replace(/(X-Amz-Signature=.)([^&]*)/, '$1...')
  debug('callAws')(`Making request to ${sanitizedUrl}`)
  got.stream(url).on('error', err => { error = err }).pipe(outputStream)
}

/**
 * Writes down all the temp files for ffmpeg to read in.
 * Returns the text filename.
 */
exports.createManifest = parts => {
  let txtFile = tempfile('.txt')
  debug('createManifest')(`Creating ${txtFile} for manifest`)
  let contents = parts.map(info => {
    return `file '${info.tempfile}'`
  }).join('\n')
  debug('createManifest')(`Writing manifest contents:\n${contents}`)
  fs.writeFileSync(txtFile, contents, 'utf8')
  return txtFile
}

/**
 * Create an AWS Polly instance.
 */
exports.createPolly = opts => {
  debug('createPolly')(`Creating Polly instance in ${opts.region}`)
  return new Polly({
    apiVersion: '2016-06-10',
    region: opts.region,
    accessKeyId: opts['access-key'],
    secretAccessKey: opts['secret-key']
  })
}

/**
 * Calls the API for each text part (throttled). Returns a Promise.
 */
exports.generateAll = (parts, opts, func, task) => {
  let count = parts.length
  task.title = `Convert to audio (0/${count})`
  return (new Promise((resolve, reject) => {
    debug('generateAll')(`Requesting ${count} audio segments, ${opts.limit} at a time`)
    async.eachOfLimit(
      parts,
      opts.limit,
      func,
      err => {
        debug('generateAll')(`Requested all parts, with error ${err}`)
        if (err) {
          return reject(err)
        }
        task.title = task.title.replace(/\d+\//, `${count}/`)
        resolve(parts)
      }
    )
  }))
}

/**
 * Returns a Promise with the temporary audio file.
 */
exports.generateSpeech = (ctx, task) => {
  const strParts = ctx.parts

  // Add in the default options.
  /* istanbul ignore next */
  ctx.opts = Object.assign({}, {
    'access-key': ctx.args.accessKey,
    ffmpeg: ctx.args.ffmpeg || 'ffmpeg',
    format: ctx.args.format || 'mp3',
    lexicon: ctx.args.lexicon,
    limit: Number(ctx.args.throttle) || 5, // eslint-disable-line no-magic-numbers
    region: ctx.args.region || 'us-east-1',
    'sample-rate': ctx.args.sampleRate,
    'secret-key': ctx.args.secretKey,
    type: ctx.args.type || 'text',
    voice: ctx.args.voice || 'Joanna'
  }, ctx.args)
  /* istanbul ignore next */
  if (typeof ctx.opts.lexicon !== 'undefined' && !Array.isArray(ctx.opts.lexicon)) {
    ctx.opts.lexicon = [ctx.opts.lexicon]
  }
  /* istanbul ignore next */
  debug('generateSpeech')(`Options: ${JSON.stringify(sanitizeOpts(ctx.opts))}`)

  /* istanbul ignore next */
  let polly = createPolly(ctx.opts)

  // Compile the text parts and options together in a packet.
  /* istanbul ignore next */
  let parts = strParts.map(part => exports.buildInfo(part, polly.getSynthesizeSpeechUrl.bind(polly), task, ctx.opts))

  /* istanbul ignore next */
  return generateAll(parts, ctx.opts, exports.callAws, task)
    .then(createManifest)
    .then(manifest => {
      ctx.manifestFile = manifest
    })
}<commit_msg>fix(generate-speech): Fix some typos in generate-speech module<commit_after>const async = require('async')
const Polly = require('aws-sdk/clients/polly').Presigner
const debug = require('debug')
const fs = require('fs-extra')
const got = require('got')
const tempfile = require('tempfile')
const fileExtensions = require('./file-extensions')
const { sanitizeOpts } = require('./sanitize-opts')

/**
 * Creates an object containing all the data.
 */
exports.buildInfo = (text, urlCreator, task, opts) => {
  return {
    opts: opts,
    task: task,
    tempfile: tempfile(`.${fileExtensions[opts.format]}`),
    text: text,
    urlcreator: urlCreator
  }
}

/**
 * Calls AWS Polly with the given info.
 */
exports.callAws = (info, i, callback) => {
  const secsPerMin = 60
  const minsInHalfHour = 30
  const halfHour = secsPerMin * minsInHalfHour

  info.task.title = info.task.title.replace(/\d+\//, `${i}/`)

  let url = info.urlcreator({
    LexiconNames: info.opts.lexicon,
    OutputFormat: info.opts.format,
    SampleRate: info.opts['sample-rate'] ? String(info.opts['sample-rate']) : undefined,
    Text: info.text,
    TextType: info.opts.type,
    VoiceId: info.opts.voice
  }, halfHour)

  let error
  debug('callAws')(`Opening output stream to ${info.tempfile}`)
  let outputStream = fs.createWriteStream(info.tempfile)
  outputStream.on('close', () => {
    debug('callAws')('Closing output stream')
    callback(error)
    /* istanbul ignore if */
    if (error) {
      debug('callAws')(`Error during request: ${error.message}`)
      // Get the error message from Amazon.
      try {
        let response = fs.readFileSync(info.tempfile, 'utf8')
        debug('callAws')(`Amazon responded with ${response}`)
        let parsedResponse = JSON.parse(response)
        error.message += `: ${parsedResponse.message}`
      } catch (err) { }
    }
  })
  let sanitizedUrl = url
    .replace(/(Text=.{1,10})([^&]*)/, '$1...')
    .replace(/(X-Amz-Credential=)([^&]*)/, '$1...')
    .replace(/(X-Amz-Signature=.)([^&]*)/, '$1...')
  debug('callAws')(`Making request to ${sanitizedUrl}`)
  got.stream(url).on('error', err => { error = err }).pipe(outputStream)
}

/**
 * Writes down all the temp files for ffmpeg to read in.
 * Returns the text filename.
 */
exports.createManifest = parts => {
  let txtFile = tempfile('.txt')
  debug('createManifest')(`Creating ${txtFile} for manifest`)
  let contents = parts.map(info => {
    return `file '${info.tempfile}'`
  }).join('\n')
  debug('createManifest')(`Writing manifest contents:\n${contents}`)
  fs.writeFileSync(txtFile, contents, 'utf8')
  return txtFile
}

/**
 * Create an AWS Polly instance.
 */
exports.createPolly = opts => {
  debug('createPolly')(`Creating Polly instance in ${opts.region}`)
  return new Polly({
    apiVersion: '2016-06-10',
    region: opts.region,
    accessKeyId: opts['access-key'],
    secretAccessKey: opts['secret-key']
  })
}

/**
 * Calls the API for each text part (throttled). Returns a Promise.
 */
exports.generateAll = (parts, opts, func, task) => {
  let count = parts.length
  task.title = `Convert to audio (0/${count})`
  return (new Promise((resolve, reject) => {
    debug('generateAll')(`Requesting ${count} audio segments, ${opts.limit} at a time`)
    async.eachOfLimit(
      parts,
      opts.limit,
      func,
      err => {
        debug('generateAll')(`Requested all parts, with error ${err}`)
        if (err) {
          return reject(err)
        }
        task.title = task.title.replace(/\d+\//, `${count}/`)
        resolve(parts)
      }
    )
  }))
}

/**
 * Returns a Promise with the temporary audio file.
 */
exports.generateSpeech = (ctx, task) => {
  const strParts = ctx.parts

  // Add in the default options.
  /* istanbul ignore next */
  ctx.opts = Object.assign({}, {
    'access-key': ctx.args.accessKey,
    ffmpeg: ctx.args.ffmpeg || 'ffmpeg',
    format: ctx.args.format || 'mp3',
    lexicon: ctx.args.lexicon,
    limit: Number(ctx.args.throttle) || 5, // eslint-disable-line no-magic-numbers
    region: ctx.args.region || 'us-east-1',
    'sample-rate': ctx.args.sampleRate,
    'secret-key': ctx.args.secretKey,
    type: ctx.args.type || 'text',
    voice: ctx.args.voice || 'Joanna'
  }, ctx.args)
  /* istanbul ignore next */
  if (typeof ctx.opts.lexicon !== 'undefined' && !Array.isArray(ctx.opts.lexicon)) {
    ctx.opts.lexicon = [ctx.opts.lexicon]
  }
  /* istanbul ignore next */
  debug('generateSpeech')(`Options: ${JSON.stringify(sanitizeOpts(ctx.opts))}`)

  /* istanbul ignore next */
  let polly = exports.createPolly(ctx.opts)

  // Compile the text parts and options together in a packet.
  /* istanbul ignore next */
  let parts = strParts.map(part => exports.buildInfo(part, polly.getSynthesizeSpeechUrl.bind(polly), task, ctx.opts))

  /* istanbul ignore next */
  return exports.generateAll(parts, ctx.opts, exports.callAws, task)
    .then(exports.createManifest)
    .then(manifest => {
      ctx.manifestFile = manifest
    })
}
{'fraction_non_alphanumeric': 0.1712846347607053, 'fraction_numerical': 0.004836272040302267, 'mean_word_length': 3.2932525951557095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7423749', 'n_tokens_mistral': 3335, 'n_tokens_neox': 3305, 'n_words': 758}