<commit_before>_ = require("lodash")
JobProcessor = require "./job.processor"
RequestProcessor = require "../request.processor"

MAX_DEQUEUE_COUNT = process.env.MAX_DEQUEUE_COUNT

_normalizeHeaders = (headers) ->
  _(headers)
    .map ({ Key, Value }) -> [ Key, Value ]
    .fromPairs()
    .value()

_normalizeMessage = ({ JobId, Resource, Method, Body, HeadersForRequest }) ->
  _.omitBy {
    method: Method
    body: Body
    resource: Resource
    headers: _normalizeHeaders HeadersForRequest
    jobId: JobId
  }, _.isUndefined

module.exports = ({ buildOpts, maxRetries, nonRetryable = [400] }) ->
  __buildOpts = (message) ->
    _.merge { resolveWithFullResponse: yes }, buildOpts(message)

  processor = RequestProcessor __buildOpts
  jobProcesor = new JobProcessor({ processor, maxRetries, nonRetryable })
<commit_msg>Removing environment variable unused<commit_after>_ = require("lodash")
JobProcessor = require "./job.processor"
RequestProcessor = require "../request.processor"

_normalizeHeaders = (headers) ->
  _(headers)
    .map ({ Key, Value }) -> [ Key, Value ]
    .fromPairs()
    .value()

_normalizeMessage = ({ JobId, Resource, Method, Body, HeadersForRequest }) ->
  _.omitBy {
    method: Method
    body: Body
    resource: Resource
    headers: _normalizeHeaders HeadersForRequest
    jobId: JobId
  }, _.isUndefined

module.exports = ({ buildOpts, maxRetries, nonRetryable = [400] }) ->
  __buildOpts = (message) ->
    _.merge { resolveWithFullResponse: yes }, buildOpts(message)

  processor = RequestProcessor __buildOpts
  jobProcesor = new JobProcessor({ processor, maxRetries, nonRetryable })

{'fraction_non_alphanumeric': 0.12331288343558282, 'fraction_numerical': 0.0036809815950920245, 'mean_word_length': 4.065217391304348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '222882', 'n_tokens_mistral': 494, 'n_tokens_neox': 495, 'n_words': 119}