<commit_before>'use strict';

var _ = require('lodash');
var AWS = require('aws-sdk');

module.exports = function() {
  var dynamoDb = new AWS.DynamoDB();

  var dynamoDbQueueSource = { };

  dynamoDbQueueSource._processResult = function(err, data, task) {
    // Did we have an error? If so, pass it through to the callback, indicate
    // that we should stop iterating, and return
    if (err) {
      task.error = err;
      task.callback(err);
      return;
    }

    // Push the items from the scan into the queue and update the total item count
    task.queue.push(data.Items);
    task.results.itemCount += data.Items.length;

    // Are we done? If so, trigger the callback for the scan and return
    if (!data.LastEvaluatedKey) {
      task.hasMoreItems = false;
      task.callback(null, task.results);
      return;
    }

    // If we have more rows to scan, repeat the scan with the new exclusive
    // start key
    task.options.ExclusiveStartKey = data.LastEvaluatedKey;
    setImmediate(task.executor);
  };

  dynamoDbQueueSource._executeScanTask = function(scanTask) {
    dynamoDb.scan(scanTask.options, scanTask.processor);
  };

  dynamoDbQueueSource._executeQueryTask = function(queryTask) {
    dynamoDb.query(queryTask.options, queryTask.processor);
  };

  dynamoDbQueueSource._isRunning = function(scanTask) {
    return (
    !scanTask.error &&
    (!scanTask.queue.idle() || scanTask.hasMoreItems)
    );
  };

  dynamoDbQueueSource._createTask = function(queue, options, callback, executor) {
    var task = {
      queue: queue,
      options: options,<commit_msg>Add DynamoDB as injected dependency<commit_after>'use strict';

var _ = require('lodash');

module.exports = function(dynamoDb) {
  var dynamoDbQueueSource = { };

  dynamoDbQueueSource._processResult = function(err, data, task) {
    // Did we have an error? If so, pass it through to the callback, indicate
    // that we should stop iterating, and return
    if (err) {
      task.error = err;
      task.callback(err);
      return;
    }

    // Push the items from the scan into the queue and update the total item
    // count
    task.queue.push(data.Items);
    task.results.itemCount += data.Items.length;

    // Are we done? If so, trigger the callback for the scan and return
    if (!data.LastEvaluatedKey) {
      task.hasMoreItems = false;
      task.callback(null, task.results);
      return;
    }

    // If we have more rows to scan, repeat the scan with the new exclusive
    // start key
    task.options.ExclusiveStartKey = data.LastEvaluatedKey;
    setImmediate(task.executor);
  };

  dynamoDbQueueSource._executeScanTask = function(scanTask) {
    dynamoDb.scan(scanTask.options, scanTask.processor);
  };

  dynamoDbQueueSource._executeQueryTask = function(queryTask) {
    dynamoDb.query(queryTask.options, queryTask.processor);
  };

  dynamoDbQueueSource._isRunning = function(scanTask) {
    return (
    !scanTask.error &&
    (!scanTask.queue.idle() || scanTask.hasMoreItems)
    );
  };

  dynamoDbQueueSource._createTask = function(
    queue,
    options,
    callback,
    executor
  ) {
    var task = {
      queue: queue,
      options: options,
{'fraction_non_alphanumeric': 0.10800627943485086, 'fraction_numerical': 0.0, 'mean_word_length': 3.074168797953964, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1268916', 'n_tokens_mistral': 981, 'n_tokens_neox': 973, 'n_words': 253}