<commit_before>
const Q = require('q')
const debug = require('debug')('qdone:enqueue')
const chalk = require('chalk')
const uuid = require('uuid')
const qrlCache = require('./qrlCache')
const AWS = require('aws-sdk')

function createFailQueue (fqueue, fqname, deadLetterTargetArn, options) {
  debug('createFailQueue(', fqueue, fqname, ')')
  const sqs = new AWS.SQS()
  const params = {
    /* Attributes: {MessageRetentionPeriod: '259200', RedrivePolicy: '{"deadLetterTargetArn": "arn:aws:sqs:us-east-1:80398EXAMPLE:MyDeadLetterQueue", "maxReceiveCount": "1000"}'}, */
    Attributes: {},
    QueueName: fqname
  }
  if (options.fifo) params.Attributes.FifoQueue = 'true'
  return sqs
    .createQueue(params)
    .promise()
    .then(function (data) {
      debug('createQueue returned', data)
      return data.QueueUrl
    })
}

function createQueue (queue, qname, deadLetterTargetArn, options) {
  debug('createQueue(', queue, qname, ')')
  const sqs = new AWS.SQS()
  const params = {
    Attributes: {
      // MessageRetentionPeriod: '259200',
      RedrivePolicy: `{"deadLetterTargetArn": "${deadLetterTargetArn}", "maxReceiveCount": "1"}'}`
    },
    QueueName: qname
  }
  if (options.fifo) params.Attributes.FifoQueue = 'true'
  return sqs
    .createQueue(params)
    .promise()
    .then(function (data) {
      debug('createQueue returned', data)
      return data.QueueUrl
    })
}

function getQueueAttributes (qrl) {
  debug('getQueueAttributes(', qrl, ')')
  const sqs = new AWS.SQS()
  return sqs
    .getQueueAttributes({
      AttributeNames: ['All'],
      QueueUrl: qrl
    })
    .promise()
    .then(function (data) {
      debug('getQueueAttributes returned', data)
      return data
    })
}

function formatMessage (command, id) {
  const message = {
    /*
    MessageAttributes: {
      City: { DataType: 'String', StringValue: 'Any City' },
      Population: { DataType: 'Number', StringValue: '1250800' }
    },
    */
    MessageBody: command
  }
  if (typeof id !== 'undefined') message.Id = '' + id
  return message
}

function sendMessage (qrl, command, options) {
  debug('sendMessage(', qrl, command, ')')
  const message = Object.assign({QueueUrl: qrl}, formatMessage(command))
  // Add in group id if we're using fifo
  if (options.fifo) {
    message.MessageGroupId = options['group-id-per-message'] ? uuid.v1() : options['group-id']
    message.MessageDeduplicationId = uuid.v1()
  }
  const sqs = new AWS.SQS()
  return sqs
    .sendMessage(message)
    .promise()
    .then(function (data) {
      debug('sendMessage returned', data)
      return data
    })
}

function sendMessageBatch (qrl, messages, options) {
  debug('sendMessageBatch(', qrl, messages.map(e => Object.assign(Object.assign({}, e), {MessageBody: e.MessageBody.slice(0, 10) + '...'})), ')')
  const params = { Entries: messages, QueueUrl: qrl }
  // Add in group id if we're using fifo
  if (options.fifo) {
    params.Entries = params.Entries.map(
      message => Object.assign({
        MessageGroupId: options['group-id-per-message'] ? uuid.v1() : options['group-id'],
        MessageDeduplicationId: uuid.v1()
      }, message)
    )
  }
  const sqs = new AWS.SQS()
  return sqs
    .sendMessageBatch(params)
    .promise()
    .then(function (data) {<commit_msg>remove unecessary ternary<commit_after>
const Q = require('q')
const debug = require('debug')('qdone:enqueue')
const chalk = require('chalk')
const uuid = require('uuid')
const qrlCache = require('./qrlCache')
const AWS = require('aws-sdk')

function createFailQueue (fqueue, fqname, deadLetterTargetArn, options) {
  debug('createFailQueue(', fqueue, fqname, ')')
  const sqs = new AWS.SQS()
  const params = {
    /* Attributes: {MessageRetentionPeriod: '259200', RedrivePolicy: '{"deadLetterTargetArn": "arn:aws:sqs:us-east-1:80398EXAMPLE:MyDeadLetterQueue", "maxReceiveCount": "1000"}'}, */
    Attributes: {},
    QueueName: fqname
  }
  if (options.fifo) params.Attributes.FifoQueue = 'true'
  return sqs
    .createQueue(params)
    .promise()
    .then(function (data) {
      debug('createQueue returned', data)
      return data.QueueUrl
    })
}

function createQueue (queue, qname, deadLetterTargetArn, options) {
  debug('createQueue(', queue, qname, ')')
  const sqs = new AWS.SQS()
  const params = {
    Attributes: {
      // MessageRetentionPeriod: '259200',
      RedrivePolicy: `{"deadLetterTargetArn": "${deadLetterTargetArn}", "maxReceiveCount": "1"}'}`
    },
    QueueName: qname
  }
  if (options.fifo) params.Attributes.FifoQueue = 'true'
  return sqs
    .createQueue(params)
    .promise()
    .then(function (data) {
      debug('createQueue returned', data)
      return data.QueueUrl
    })
}

function getQueueAttributes (qrl) {
  debug('getQueueAttributes(', qrl, ')')
  const sqs = new AWS.SQS()
  return sqs
    .getQueueAttributes({
      AttributeNames: ['All'],
      QueueUrl: qrl
    })
    .promise()
    .then(function (data) {
      debug('getQueueAttributes returned', data)
      return data
    })
}

function formatMessage (command, id) {
  const message = {
    /*
    MessageAttributes: {
      City: { DataType: 'String', StringValue: 'Any City' },
      Population: { DataType: 'Number', StringValue: '1250800' }
    },
    */
    MessageBody: command
  }
  if (typeof id !== 'undefined') message.Id = '' + id
  return message
}

function sendMessage (qrl, command, options) {
  debug('sendMessage(', qrl, command, ')')
  const message = Object.assign({QueueUrl: qrl}, formatMessage(command))
  // Add in group id if we're using fifo
  if (options.fifo) {
    message.MessageGroupId = options['group-id']
    message.MessageDeduplicationId = uuid.v1()
  }
  const sqs = new AWS.SQS()
  return sqs
    .sendMessage(message)
    .promise()
    .then(function (data) {
      debug('sendMessage returned', data)
      return data
    })
}

function sendMessageBatch (qrl, messages, options) {
  debug('sendMessageBatch(', qrl, messages.map(e => Object.assign(Object.assign({}, e), {MessageBody: e.MessageBody.slice(0, 10) + '...'})), ')')
  const params = { Entries: messages, QueueUrl: qrl }
  // Add in group id if we're using fifo
  if (options.fifo) {
    params.Entries = params.Entries.map(
      message => Object.assign({
        MessageGroupId: options['group-id-per-message'] ? uuid.v1() : options['group-id'],
        MessageDeduplicationId: uuid.v1()
      }, message)
    )
  }
  const sqs = new AWS.SQS()
  return sqs
    .sendMessageBatch(params)
    .promise()
    .then(function (data) {
{'fraction_non_alphanumeric': 0.15238678090575275, 'fraction_numerical': 0.01116891064871481, 'mean_word_length': 3.4866163349347974, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '177139', 'n_tokens_mistral': 2226, 'n_tokens_neox': 2131, 'n_words': 492}